diff --git "a/checkpoint-19000/trainer_state.json" "b/checkpoint-19000/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-19000/trainer_state.json" @@ -0,0 +1,141427 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.3773285141200302, + "global_step": 19000, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-07, + "loss": 10.8568, + "theoretical_loss": 20.812814784551147, + "tokens_seen": 65536 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-06, + "loss": 10.8727, + "theoretical_loss": 17.56619808733264, + "tokens_seen": 131072 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904763e-06, + "loss": 10.7356, + "theoretical_loss": 15.939474075840563, + "tokens_seen": 196608 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-06, + "loss": 10.7113, + "theoretical_loss": 14.892313738992565, + "tokens_seen": 262144 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-06, + "loss": 10.3498, + "theoretical_loss": 14.13621392076697, + "tokens_seen": 327680 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809525e-06, + "loss": 10.1826, + "theoretical_loss": 13.552558455554218, + "tokens_seen": 393216 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-06, + "loss": 10.0176, + "theoretical_loss": 13.081805984405184, + "tokens_seen": 458752 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507936e-06, + "loss": 9.6515, + "theoretical_loss": 12.690126608487319, + "tokens_seen": 524288 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571428e-06, + "loss": 9.2598, + "theoretical_loss": 12.356589446877619, + "tokens_seen": 589824 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-06, + "loss": 9.1219, + "theoretical_loss": 12.067409590039073, + "tokens_seen": 655360 + }, + { + "epoch": 0.0, + "learning_rate": 1.0912698412698412e-05, + "loss": 9.2934, + "theoretical_loss": 11.813063214105672, + "tokens_seen": 720896 + }, + { + "epoch": 0.0, + "learning_rate": 1.1904761904761905e-05, + "loss": 9.0973, + "theoretical_loss": 11.586716191710725, + "tokens_seen": 786432 + }, + { + "epoch": 0.0, + "learning_rate": 1.2896825396825396e-05, + "loss": 9.4804, + "theoretical_loss": 11.383311123190783, + "tokens_seen": 851968 + }, + { + "epoch": 0.0, + "learning_rate": 1.3888888888888888e-05, + "loss": 9.3208, + "theoretical_loss": 11.199008685115867, + "tokens_seen": 917504 + }, + { + "epoch": 0.0, + "learning_rate": 1.4880952380952381e-05, + "loss": 9.1864, + "theoretical_loss": 11.030830900981908, + "tokens_seen": 983040 + }, + { + "epoch": 0.0, + "learning_rate": 1.5873015873015872e-05, + "loss": 9.398, + "theoretical_loss": 10.876425069460945, + "tokens_seen": 1048576 + }, + { + "epoch": 0.0, + "learning_rate": 1.6865079365079364e-05, + "loss": 8.9394, + "theoretical_loss": 10.73390272306672, + "tokens_seen": 1114112 + }, + { + "epoch": 0.0, + "learning_rate": 1.7857142857142855e-05, + "loss": 9.0636, + "theoretical_loss": 10.601726859234274, + "tokens_seen": 1179648 + }, + { + "epoch": 0.0, + "learning_rate": 1.884920634920635e-05, + "loss": 9.1583, + "theoretical_loss": 10.478631155360636, + "tokens_seen": 1245184 + }, + { + "epoch": 0.0, + "learning_rate": 1.984126984126984e-05, + "loss": 9.1124, + "theoretical_loss": 10.363560926767326, + "tokens_seen": 1310720 + }, + { + "epoch": 0.0, + "learning_rate": 2.0833333333333333e-05, + "loss": 9.0087, + "theoretical_loss": 10.25562920390074, + "tokens_seen": 1376256 + }, + { + "epoch": 0.0, + "learning_rate": 2.1825396825396824e-05, + "loss": 9.0788, + "theoretical_loss": 10.154083536274014, + "tokens_seen": 1441792 + }, + { + "epoch": 0.0, + "learning_rate": 2.2817460317460315e-05, + "loss": 8.4406, + "theoretical_loss": 10.058280544736593, + "tokens_seen": 1507328 + }, + { + "epoch": 0.0, + "learning_rate": 2.380952380952381e-05, + "loss": 8.9754, + "theoretical_loss": 9.967666161844274, + "tokens_seen": 1572864 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 12267, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 8.854658126831055, + "objective/train/theoretical_loss": 9.881760109397105, + "objective/train/tokens_used": 22098400, + "theoretical_loss": 9.881760109397105, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 2.48015873015873e-05, + "loss": 8.9164, + "theoretical_loss": 9.881760109397105, + "tokens_seen": 1638400 + }, + { + "epoch": 0.0, + "learning_rate": 2.5793650793650793e-05, + "loss": 7.9452, + "theoretical_loss": 9.800143574544556, + "tokens_seen": 1703936 + }, + { + "epoch": 0.0, + "learning_rate": 2.6785714285714284e-05, + "loss": 8.81, + "theoretical_loss": 9.722449329911441, + "tokens_seen": 1769472 + }, + { + "epoch": 0.0, + "learning_rate": 2.7777777777777776e-05, + "loss": 9.0741, + "theoretical_loss": 9.648353742085542, + "tokens_seen": 1835008 + }, + { + "epoch": 0.0, + "learning_rate": 2.876984126984127e-05, + "loss": 8.8256, + "theoretical_loss": 9.577570254149634, + "tokens_seen": 1900544 + }, + { + "epoch": 0.0, + "learning_rate": 2.9761904761904762e-05, + "loss": 8.3274, + "theoretical_loss": 9.509844029768848, + "tokens_seen": 1966080 + }, + { + "epoch": 0.0, + "learning_rate": 3.075396825396825e-05, + "loss": 8.55, + "theoretical_loss": 9.44494752063593, + "tokens_seen": 2031616 + }, + { + "epoch": 0.0, + "learning_rate": 3.1746031746031745e-05, + "loss": 8.0979, + "theoretical_loss": 9.382676773914453, + "tokens_seen": 2097152 + }, + { + "epoch": 0.0, + "learning_rate": 3.273809523809524e-05, + "loss": 8.3374, + "theoretical_loss": 9.322848337237977, + "tokens_seen": 2162688 + }, + { + "epoch": 0.0, + "learning_rate": 3.373015873015873e-05, + "loss": 8.5177, + "theoretical_loss": 9.26529664966427, + "tokens_seen": 2228224 + }, + { + "epoch": 0.0, + "learning_rate": 3.472222222222222e-05, + "loss": 8.6686, + "theoretical_loss": 9.209871830448751, + "tokens_seen": 2293760 + }, + { + "epoch": 0.0, + "learning_rate": 3.571428571428571e-05, + "loss": 8.675, + "theoretical_loss": 9.156437795512288, + "tokens_seen": 2359296 + }, + { + "epoch": 0.0, + "learning_rate": 3.670634920634921e-05, + "loss": 8.4694, + "theoretical_loss": 9.104870645417346, + "tokens_seen": 2424832 + }, + { + "epoch": 0.0, + "learning_rate": 3.76984126984127e-05, + "loss": 8.2407, + "theoretical_loss": 9.05505727953773, + "tokens_seen": 2490368 + }, + { + "epoch": 0.0, + "learning_rate": 3.8690476190476195e-05, + "loss": 8.5934, + "theoretical_loss": 9.006894199647824, + "tokens_seen": 2555904 + }, + { + "epoch": 0.0, + "learning_rate": 3.968253968253968e-05, + "loss": 7.8491, + "theoretical_loss": 8.960286472913351, + "tokens_seen": 2621440 + }, + { + "epoch": 0.0, + "learning_rate": 4.067460317460318e-05, + "loss": 7.8615, + "theoretical_loss": 8.915146829644607, + "tokens_seen": 2686976 + }, + { + "epoch": 0.0, + "learning_rate": 4.1666666666666665e-05, + "loss": 7.7589, + "theoretical_loss": 8.871394875482219, + "tokens_seen": 2752512 + }, + { + "epoch": 0.0, + "learning_rate": 4.265873015873016e-05, + "loss": 8.0014, + "theoretical_loss": 8.828956401157495, + "tokens_seen": 2818048 + }, + { + "epoch": 0.0, + "learning_rate": 4.365079365079365e-05, + "loss": 7.8309, + "theoretical_loss": 8.787762775782408, + "tokens_seen": 2883584 + }, + { + "epoch": 0.0, + "learning_rate": 4.464285714285714e-05, + "loss": 8.259, + "theoretical_loss": 8.747750411915451, + "tokens_seen": 2949120 + }, + { + "epoch": 0.0, + "learning_rate": 4.563492063492063e-05, + "loss": 8.1107, + "theoretical_loss": 8.708860292524829, + "tokens_seen": 3014656 + }, + { + "epoch": 0.0, + "learning_rate": 4.6626984126984126e-05, + "loss": 8.2947, + "theoretical_loss": 8.671037551512843, + "tokens_seen": 3080192 + }, + { + "epoch": 0.0, + "learning_rate": 4.761904761904762e-05, + "loss": 7.764, + "theoretical_loss": 8.63423110073947, + "tokens_seen": 3145728 + }, + { + "epoch": 0.0, + "learning_rate": 4.8611111111111115e-05, + "loss": 8.1678, + "theoretical_loss": 8.59839329754032, + "tokens_seen": 3211264 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 13483, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 8.108365058898926, + "objective/train/theoretical_loss": 8.563479647615063, + "objective/train/tokens_used": 23736800, + "theoretical_loss": 8.563479647615063, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 4.96031746031746e-05, + "loss": 7.5801, + "theoretical_loss": 8.563479647615063, + "tokens_seen": 3276800 + }, + { + "epoch": 0.0, + "learning_rate": 5.05952380952381e-05, + "loss": 7.8434, + "theoretical_loss": 8.52944853889911, + "tokens_seen": 3342336 + }, + { + "epoch": 0.0, + "learning_rate": 5.1587301587301586e-05, + "loss": 7.8492, + "theoretical_loss": 8.496261002649998, + "tokens_seen": 3407872 + }, + { + "epoch": 0.0, + "learning_rate": 5.257936507936508e-05, + "loss": 7.8312, + "theoretical_loss": 8.463880498501183, + "tokens_seen": 3473408 + }, + { + "epoch": 0.0, + "learning_rate": 5.357142857142857e-05, + "loss": 7.3015, + "theoretical_loss": 8.432272720676774, + "tokens_seen": 3538944 + }, + { + "epoch": 0.0, + "learning_rate": 5.4563492063492063e-05, + "loss": 7.9474, + "theoretical_loss": 8.401405422934712, + "tokens_seen": 3604480 + }, + { + "epoch": 0.0, + "learning_rate": 5.555555555555555e-05, + "loss": 7.6148, + "theoretical_loss": 8.371248260124204, + "tokens_seen": 3670016 + }, + { + "epoch": 0.0, + "learning_rate": 5.6547619047619046e-05, + "loss": 7.5108, + "theoretical_loss": 8.341772644515071, + "tokens_seen": 3735552 + }, + { + "epoch": 0.0, + "learning_rate": 5.753968253968254e-05, + "loss": 7.6168, + "theoretical_loss": 8.312951615289325, + "tokens_seen": 3801088 + }, + { + "epoch": 0.0, + "learning_rate": 5.8531746031746036e-05, + "loss": 7.8436, + "theoretical_loss": 8.284759719785177, + "tokens_seen": 3866624 + }, + { + "epoch": 0.0, + "learning_rate": 5.9523809523809524e-05, + "loss": 7.0651, + "theoretical_loss": 8.257172905255858, + "tokens_seen": 3932160 + }, + { + "epoch": 0.0, + "learning_rate": 6.051587301587302e-05, + "loss": 7.4859, + "theoretical_loss": 8.23016842005411, + "tokens_seen": 3997696 + }, + { + "epoch": 0.0, + "learning_rate": 6.15079365079365e-05, + "loss": 7.531, + "theoretical_loss": 8.203724723281965, + "tokens_seen": 4063232 + }, + { + "epoch": 0.0, + "learning_rate": 6.25e-05, + "loss": 7.5815, + "theoretical_loss": 8.177821402057042, + "tokens_seen": 4128768 + }, + { + "epoch": 0.0, + "learning_rate": 6.349206349206349e-05, + "loss": 7.2471, + "theoretical_loss": 8.152439095643611, + "tokens_seen": 4194304 + }, + { + "epoch": 0.0, + "learning_rate": 6.448412698412699e-05, + "loss": 7.4488, + "theoretical_loss": 8.127559425781495, + "tokens_seen": 4259840 + }, + { + "epoch": 0.0, + "learning_rate": 6.547619047619048e-05, + "loss": 7.4517, + "theoretical_loss": 8.103164932619705, + "tokens_seen": 4325376 + }, + { + "epoch": 0.0, + "learning_rate": 6.646825396825397e-05, + "loss": 7.1039, + "theoretical_loss": 8.079239015726635, + "tokens_seen": 4390912 + }, + { + "epoch": 0.0, + "learning_rate": 6.746031746031745e-05, + "loss": 7.1279, + "theoretical_loss": 8.055765879705412, + "tokens_seen": 4456448 + }, + { + "epoch": 0.0, + "learning_rate": 6.845238095238096e-05, + "loss": 7.3354, + "theoretical_loss": 8.032730483993003, + "tokens_seen": 4521984 + }, + { + "epoch": 0.0, + "learning_rate": 6.944444444444444e-05, + "loss": 7.1485, + "theoretical_loss": 8.01011849646583, + "tokens_seen": 4587520 + }, + { + "epoch": 0.0, + "learning_rate": 7.043650793650793e-05, + "loss": 6.9851, + "theoretical_loss": 7.987916250513374, + "tokens_seen": 4653056 + }, + { + "epoch": 0.0, + "learning_rate": 7.142857142857142e-05, + "loss": 6.9432, + "theoretical_loss": 7.966110705275796, + "tokens_seen": 4718592 + }, + { + "epoch": 0.0, + "learning_rate": 7.242063492063492e-05, + "loss": 6.9933, + "theoretical_loss": 7.944689408771984, + "tokens_seen": 4784128 + }, + { + "epoch": 0.0, + "learning_rate": 7.341269841269842e-05, + "loss": 7.0325, + "theoretical_loss": 7.923640463671514, + "tokens_seen": 4849664 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 14033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.057826519012451, + "objective/train/theoretical_loss": 7.902952495488062, + "objective/train/tokens_used": 25375200, + "theoretical_loss": 7.902952495488062, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 7.440476190476191e-05, + "loss": 6.7791, + "theoretical_loss": 7.902952495488062, + "tokens_seen": 4915200 + }, + { + "epoch": 0.0, + "learning_rate": 7.53968253968254e-05, + "loss": 6.7978, + "theoretical_loss": 7.882614622993199, + "tokens_seen": 4980736 + }, + { + "epoch": 0.0, + "learning_rate": 7.63888888888889e-05, + "loss": 6.8558, + "theoretical_loss": 7.8626164306686235, + "tokens_seen": 5046272 + }, + { + "epoch": 0.0, + "learning_rate": 7.738095238095239e-05, + "loss": 6.9141, + "theoretical_loss": 7.842947943031932, + "tokens_seen": 5111808 + }, + { + "epoch": 0.0, + "learning_rate": 7.837301587301588e-05, + "loss": 6.9255, + "theoretical_loss": 7.8235996006863076, + "tokens_seen": 5177344 + }, + { + "epoch": 0.0, + "learning_rate": 7.936507936507937e-05, + "loss": 6.7895, + "theoretical_loss": 7.8045622379581605, + "tokens_seen": 5242880 + }, + { + "epoch": 0.0, + "learning_rate": 8.035714285714287e-05, + "loss": 6.7059, + "theoretical_loss": 7.785827061999068, + "tokens_seen": 5308416 + }, + { + "epoch": 0.0, + "learning_rate": 8.134920634920635e-05, + "loss": 6.815, + "theoretical_loss": 7.767385633239359, + "tokens_seen": 5373952 + }, + { + "epoch": 0.0, + "learning_rate": 8.234126984126984e-05, + "loss": 6.8445, + "theoretical_loss": 7.749229847090614, + "tokens_seen": 5439488 + }, + { + "epoch": 0.0, + "learning_rate": 8.333333333333333e-05, + "loss": 6.7849, + "theoretical_loss": 7.731351916803314, + "tokens_seen": 5505024 + }, + { + "epoch": 0.0, + "learning_rate": 8.432539682539683e-05, + "loss": 6.5349, + "theoretical_loss": 7.7137443573939155, + "tokens_seen": 5570560 + }, + { + "epoch": 0.0, + "learning_rate": 8.531746031746032e-05, + "loss": 6.5103, + "theoretical_loss": 7.696399970562929, + "tokens_seen": 5636096 + }, + { + "epoch": 0.0, + "learning_rate": 8.630952380952381e-05, + "loss": 6.9307, + "theoretical_loss": 7.679311830532177, + "tokens_seen": 5701632 + }, + { + "epoch": 0.0, + "learning_rate": 8.73015873015873e-05, + "loss": 6.8115, + "theoretical_loss": 7.662473270735324, + "tokens_seen": 5767168 + }, + { + "epoch": 0.0, + "learning_rate": 8.82936507936508e-05, + "loss": 6.6601, + "theoretical_loss": 7.645877871301275, + "tokens_seen": 5832704 + }, + { + "epoch": 0.0, + "learning_rate": 8.928571428571429e-05, + "loss": 6.2238, + "theoretical_loss": 7.6295194472748555, + "tokens_seen": 5898240 + }, + { + "epoch": 0.0, + "learning_rate": 9.027777777777777e-05, + "loss": 6.6176, + "theoretical_loss": 7.613392037523692, + "tokens_seen": 5963776 + }, + { + "epoch": 0.0, + "learning_rate": 9.126984126984126e-05, + "loss": 6.5932, + "theoretical_loss": 7.597489894284275, + "tokens_seen": 6029312 + }, + { + "epoch": 0.0, + "learning_rate": 9.226190476190476e-05, + "loss": 6.51, + "theoretical_loss": 7.581807473303883, + "tokens_seen": 6094848 + }, + { + "epoch": 0.0, + "learning_rate": 9.325396825396825e-05, + "loss": 6.3768, + "theoretical_loss": 7.566339424538366, + "tokens_seen": 6160384 + }, + { + "epoch": 0.0, + "learning_rate": 9.424603174603175e-05, + "loss": 6.5465, + "theoretical_loss": 7.551080583368945, + "tokens_seen": 6225920 + }, + { + "epoch": 0.0, + "learning_rate": 9.523809523809524e-05, + "loss": 6.8931, + "theoretical_loss": 7.536025962303915, + "tokens_seen": 6291456 + }, + { + "epoch": 0.0, + "learning_rate": 9.623015873015874e-05, + "loss": 6.5266, + "theoretical_loss": 7.521170743133757, + "tokens_seen": 6356992 + }, + { + "epoch": 0.0, + "learning_rate": 9.722222222222223e-05, + "loss": 6.4856, + "theoretical_loss": 7.506510269510493, + "tokens_seen": 6422528 + }, + { + "epoch": 0.0, + "learning_rate": 9.821428571428572e-05, + "loss": 6.4763, + "theoretical_loss": 7.492040039924245, + "tokens_seen": 6488064 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 15357, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.699707984924316, + "objective/train/theoretical_loss": 7.4777557010520255, + "objective/train/tokens_used": 27013600, + "theoretical_loss": 7.4777557010520255, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 9.92063492063492e-05, + "loss": 6.6532, + "theoretical_loss": 7.4777557010520255, + "tokens_seen": 6553600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010019841269841271, + "loss": 6.4, + "theoretical_loss": 7.463653041455457, + "tokens_seen": 6619136 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001011904761904762, + "loss": 6.2462, + "theoretical_loss": 7.449727985605911, + "tokens_seen": 6684672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010218253968253968, + "loss": 6.6399, + "theoretical_loss": 7.435976588217015, + "tokens_seen": 6750208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010317460317460317, + "loss": 6.1887, + "theoretical_loss": 7.4223950288659, + "tokens_seen": 6815744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010416666666666667, + "loss": 6.4129, + "theoretical_loss": 7.40897960688587, + "tokens_seen": 6881280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010515873015873016, + "loss": 6.2296, + "theoretical_loss": 7.395726736514341, + "tokens_seen": 6946816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010615079365079365, + "loss": 6.3505, + "theoretical_loss": 7.3826329422810275, + "tokens_seen": 7012352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010714285714285714, + "loss": 6.5043, + "theoretical_loss": 7.369694854622368, + "tokens_seen": 7077888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010813492063492064, + "loss": 6.3689, + "theoretical_loss": 7.3569092057090835, + "tokens_seen": 7143424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00010912698412698413, + "loss": 6.5583, + "theoretical_loss": 7.3442728254746825, + "tokens_seen": 7208960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011011904761904761, + "loss": 6.3757, + "theoretical_loss": 7.3317826378335145, + "tokens_seen": 7274496 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001111111111111111, + "loss": 6.3164, + "theoretical_loss": 7.319435657077673, + "tokens_seen": 7340032 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001121031746031746, + "loss": 6.205, + "theoretical_loss": 7.307228984442819, + "tokens_seen": 7405568 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011309523809523809, + "loss": 6.4975, + "theoretical_loss": 7.29515980483356, + "tokens_seen": 7471104 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011408730158730158, + "loss": 6.087, + "theoretical_loss": 7.283225383699648, + "tokens_seen": 7536640 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011507936507936508, + "loss": 6.0785, + "theoretical_loss": 7.271423064054828, + "tokens_seen": 7602176 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011607142857142858, + "loss": 6.4949, + "theoretical_loss": 7.259750263630618, + "tokens_seen": 7667712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011706349206349207, + "loss": 6.2155, + "theoretical_loss": 7.248204472157866, + "tokens_seen": 7733248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011805555555555556, + "loss": 6.0229, + "theoretical_loss": 7.236783248769257, + "tokens_seen": 7798784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00011904761904761905, + "loss": 6.6244, + "theoretical_loss": 7.225484219516492, + "tokens_seen": 7864320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012003968253968255, + "loss": 5.9387, + "theoretical_loss": 7.2143050749961155, + "tokens_seen": 7929856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012103174603174604, + "loss": 6.2842, + "theoretical_loss": 7.2032435680783955, + "tokens_seen": 7995392 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012202380952380953, + "loss": 6.2047, + "theoretical_loss": 7.19229751173401, + "tokens_seen": 8060928 + }, + { + "epoch": 0.0, + "learning_rate": 0.000123015873015873, + "loss": 6.2571, + "theoretical_loss": 7.181464776953511, + "tokens_seen": 8126464 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 16711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.2001729011535645, + "objective/train/theoretical_loss": 7.1707432907549595, + "objective/train/tokens_used": 28652000, + "theoretical_loss": 7.1707432907549595, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001240079365079365, + "loss": 6.0533, + "theoretical_loss": 7.1707432907549595, + "tokens_seen": 8192000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000125, + "loss": 6.0526, + "theoretical_loss": 7.160131034275267, + "tokens_seen": 8257536 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001259920634920635, + "loss": 6.0532, + "theoretical_loss": 7.149626040941133, + "tokens_seen": 8323072 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012698412698412698, + "loss": 5.9719, + "theoretical_loss": 7.139226394715633, + "tokens_seen": 8388608 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012797619047619048, + "loss": 6.0403, + "theoretical_loss": 7.12893022841679, + "tokens_seen": 8454144 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012896825396825398, + "loss": 5.7847, + "theoretical_loss": 7.118735722104612, + "tokens_seen": 8519680 + }, + { + "epoch": 0.0, + "learning_rate": 0.00012996031746031748, + "loss": 5.9094, + "theoretical_loss": 7.10864110153334, + "tokens_seen": 8585216 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013095238095238096, + "loss": 6.0872, + "theoretical_loss": 7.098644636665766, + "tokens_seen": 8650752 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013194444444444446, + "loss": 6.0185, + "theoretical_loss": 7.088744640246688, + "tokens_seen": 8716288 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013293650793650793, + "loss": 6.0768, + "theoretical_loss": 7.078939466432745, + "tokens_seen": 8781824 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013392857142857144, + "loss": 6.3022, + "theoretical_loss": 7.069227509475962, + "tokens_seen": 8847360 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001349206349206349, + "loss": 6.0881, + "theoretical_loss": 7.059607202458563, + "tokens_seen": 8912896 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001359126984126984, + "loss": 5.7381, + "theoretical_loss": 7.0500770160766635, + "tokens_seen": 8978432 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001369047619047619, + "loss": 5.9916, + "theoretical_loss": 7.04063545747062, + "tokens_seen": 9043968 + }, + { + "epoch": 0.0, + "learning_rate": 0.00013789682539682541, + "loss": 6.0983, + "theoretical_loss": 7.031281069099929, + "tokens_seen": 9109504 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001388888888888889, + "loss": 6.0662, + "theoretical_loss": 7.022012427660673, + "tokens_seen": 9175040 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001398809523809524, + "loss": 5.7078, + "theoretical_loss": 7.012828143043604, + "tokens_seen": 9240576 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014087301587301586, + "loss": 6.2455, + "theoretical_loss": 7.003726857331067, + "tokens_seen": 9306112 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014186507936507937, + "loss": 5.9609, + "theoretical_loss": 6.994707243831053, + "tokens_seen": 9371648 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014285714285714284, + "loss": 5.9423, + "theoretical_loss": 6.985768006146776, + "tokens_seen": 9437184 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014384920634920634, + "loss": 6.0329, + "theoretical_loss": 6.976907877280185, + "tokens_seen": 9502720 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014484126984126984, + "loss": 5.7799, + "theoretical_loss": 6.96812561876801, + "tokens_seen": 9568256 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014583333333333335, + "loss": 5.9063, + "theoretical_loss": 6.95942001984889, + "tokens_seen": 9633792 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014682539682539685, + "loss": 5.9317, + "theoretical_loss": 6.950789896660304, + "tokens_seen": 9699328 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014781746031746032, + "loss": 5.8925, + "theoretical_loss": 6.942234091464025, + "tokens_seen": 9764864 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 17166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 6.479184627532959, + "objective/train/theoretical_loss": 6.933751471898896, + "objective/train/tokens_used": 30290400, + "theoretical_loss": 6.933751471898896, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.00014880952380952382, + "loss": 6.2539, + "theoretical_loss": 6.933751471898896, + "tokens_seen": 9830400 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001498015873015873, + "loss": 5.996, + "theoretical_loss": 6.925340930259813, + "tokens_seen": 9895936 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001507936507936508, + "loss": 5.8197, + "theoretical_loss": 6.917001382801793, + "tokens_seen": 9961472 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015178571428571427, + "loss": 6.0534, + "theoretical_loss": 6.908731769068142, + "tokens_seen": 10027008 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001527777777777778, + "loss": 5.7838, + "theoretical_loss": 6.900531051241684, + "tokens_seen": 10092544 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015376984126984128, + "loss": 5.7799, + "theoretical_loss": 6.892398213518156, + "tokens_seen": 10158080 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015476190476190478, + "loss": 6.1056, + "theoretical_loss": 6.884332261500866, + "tokens_seen": 10223616 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015575396825396825, + "loss": 5.5432, + "theoretical_loss": 6.876332221615746, + "tokens_seen": 10289152 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015674603174603175, + "loss": 5.8572, + "theoretical_loss": 6.8683971405459925, + "tokens_seen": 10354688 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015773809523809523, + "loss": 6.226, + "theoretical_loss": 6.860526084685546, + "tokens_seen": 10420224 + }, + { + "epoch": 0.0, + "learning_rate": 0.00015873015873015873, + "loss": 5.7117, + "theoretical_loss": 6.852718139610625, + "tokens_seen": 10485760 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001597222222222222, + "loss": 5.6684, + "theoretical_loss": 6.844972409568637, + "tokens_seen": 10551296 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016071428571428573, + "loss": 5.8166, + "theoretical_loss": 6.8372880169837895, + "tokens_seen": 10616832 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001617063492063492, + "loss": 5.971, + "theoretical_loss": 6.829664101978745, + "tokens_seen": 10682368 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001626984126984127, + "loss": 5.7343, + "theoretical_loss": 6.8220998219117135, + "tokens_seen": 10747904 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016369047619047618, + "loss": 5.8061, + "theoretical_loss": 6.81459435092839, + "tokens_seen": 10813440 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016468253968253969, + "loss": 6.0029, + "theoretical_loss": 6.807146879528176, + "tokens_seen": 10878976 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016567460317460316, + "loss": 5.5367, + "theoretical_loss": 6.79975661414414, + "tokens_seen": 10944512 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016666666666666666, + "loss": 5.6256, + "theoretical_loss": 6.79242277673622, + "tokens_seen": 11010048 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016765873015873016, + "loss": 5.7868, + "theoretical_loss": 6.785144604397143, + "tokens_seen": 11075584 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016865079365079366, + "loss": 5.6046, + "theoretical_loss": 6.777921348970633, + "tokens_seen": 11141120 + }, + { + "epoch": 0.0, + "learning_rate": 0.00016964285714285717, + "loss": 5.9753, + "theoretical_loss": 6.770752276681419, + "tokens_seen": 11206656 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017063492063492064, + "loss": 5.9022, + "theoretical_loss": 6.76363666777662, + "tokens_seen": 11272192 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017162698412698414, + "loss": 5.8921, + "theoretical_loss": 6.756573816178118, + "tokens_seen": 11337728 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017261904761904762, + "loss": 5.9722, + "theoretical_loss": 6.749563029145481, + "tokens_seen": 11403264 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 18369, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.672413349151611, + "objective/train/theoretical_loss": 6.742603626949085, + "objective/train/tokens_used": 31928800, + "theoretical_loss": 6.742603626949085, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017361111111111112, + "loss": 5.4506, + "theoretical_loss": 6.742603626949085, + "tokens_seen": 11468800 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001746031746031746, + "loss": 6.0078, + "theoretical_loss": 6.735694942553071, + "tokens_seen": 11534336 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001755952380952381, + "loss": 5.8091, + "theoretical_loss": 6.728836321307757, + "tokens_seen": 11599872 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001765873015873016, + "loss": 5.6113, + "theoretical_loss": 6.722027120651221, + "tokens_seen": 11665408 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001775793650793651, + "loss": 5.6758, + "theoretical_loss": 6.715266709819684, + "tokens_seen": 11730944 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017857142857142857, + "loss": 5.6378, + "theoretical_loss": 6.708554469566408, + "tokens_seen": 11796480 + }, + { + "epoch": 0.0, + "learning_rate": 0.00017956349206349207, + "loss": 5.7411, + "theoretical_loss": 6.701889791888819, + "tokens_seen": 11862016 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018055555555555555, + "loss": 5.8311, + "theoretical_loss": 6.695272079763555, + "tokens_seen": 11927552 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018154761904761905, + "loss": 5.2846, + "theoretical_loss": 6.688700746889175, + "tokens_seen": 11993088 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018253968253968252, + "loss": 5.7687, + "theoretical_loss": 6.682175217436269, + "tokens_seen": 12058624 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018353174603174602, + "loss": 5.7687, + "theoretical_loss": 6.675694925804711, + "tokens_seen": 12124160 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018452380952380953, + "loss": 5.6209, + "theoretical_loss": 6.66925931638781, + "tokens_seen": 12189696 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018551587301587303, + "loss": 5.9193, + "theoretical_loss": 6.662867843343154, + "tokens_seen": 12255232 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001865079365079365, + "loss": 5.6754, + "theoretical_loss": 6.6565199703698745, + "tokens_seen": 12320768 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001875, + "loss": 5.8597, + "theoretical_loss": 6.6502151704921655, + "tokens_seen": 12386304 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001884920634920635, + "loss": 5.709, + "theoretical_loss": 6.643952925848826, + "tokens_seen": 12451840 + }, + { + "epoch": 0.0, + "learning_rate": 0.00018948412698412698, + "loss": 5.7054, + "theoretical_loss": 6.637732727488622, + "tokens_seen": 12517376 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019047619047619048, + "loss": 5.5972, + "theoretical_loss": 6.6315540751713, + "tokens_seen": 12582912 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019146825396825398, + "loss": 5.5887, + "theoretical_loss": 6.625416477174044, + "tokens_seen": 12648448 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019246031746031748, + "loss": 5.7082, + "theoretical_loss": 6.619319450103218, + "tokens_seen": 12713984 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019345238095238096, + "loss": 5.5405, + "theoretical_loss": 6.613262518711206, + "tokens_seen": 12779520 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019444444444444446, + "loss": 5.5806, + "theoretical_loss": 6.607245215718208, + "tokens_seen": 12845056 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019543650793650793, + "loss": 5.4612, + "theoretical_loss": 6.601267081638806, + "tokens_seen": 12910592 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019642857142857144, + "loss": 5.6405, + "theoretical_loss": 6.595327664613176, + "tokens_seen": 12976128 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001974206349206349, + "loss": 5.3968, + "theoretical_loss": 6.58942652024278, + "tokens_seen": 13041664 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 19017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.398904323577881, + "objective/train/theoretical_loss": 6.583563211430409, + "objective/train/tokens_used": 33567200, + "theoretical_loss": 6.583563211430409, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.0001984126984126984, + "loss": 5.5392, + "theoretical_loss": 6.583563211430409, + "tokens_seen": 13107200 + }, + { + "epoch": 0.0, + "learning_rate": 0.00019940476190476191, + "loss": 5.3995, + "theoretical_loss": 6.577737308224426, + "tokens_seen": 13172736 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020039682539682542, + "loss": 5.5665, + "theoretical_loss": 6.571948387667094, + "tokens_seen": 13238272 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002013888888888889, + "loss": 5.5745, + "theoretical_loss": 6.566196033646859, + "tokens_seen": 13303808 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002023809523809524, + "loss": 5.4756, + "theoretical_loss": 6.560479836754459, + "tokens_seen": 13369344 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020337301587301587, + "loss": 5.7895, + "theoretical_loss": 6.554799394142741, + "tokens_seen": 13434880 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020436507936507937, + "loss": 5.3965, + "theoretical_loss": 6.549154309390087, + "tokens_seen": 13500416 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020535714285714284, + "loss": 5.494, + "theoretical_loss": 6.5435441923673245, + "tokens_seen": 13565952 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020634920634920634, + "loss": 5.5559, + "theoretical_loss": 6.537968659108021, + "tokens_seen": 13631488 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020734126984126985, + "loss": 5.9125, + "theoretical_loss": 6.532427331682063, + "tokens_seen": 13697024 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020833333333333335, + "loss": 5.5235, + "theoretical_loss": 6.5269198380724145, + "tokens_seen": 13762560 + }, + { + "epoch": 0.0, + "learning_rate": 0.00020932539682539685, + "loss": 5.6307, + "theoretical_loss": 6.5214458120549725, + "tokens_seen": 13828096 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021031746031746032, + "loss": 5.3661, + "theoretical_loss": 6.516004893081412, + "tokens_seen": 13893632 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021130952380952382, + "loss": 5.2823, + "theoretical_loss": 6.510596726164945, + "tokens_seen": 13959168 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002123015873015873, + "loss": 5.4241, + "theoretical_loss": 6.505220961768906, + "tokens_seen": 14024704 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002132936507936508, + "loss": 5.3679, + "theoretical_loss": 6.499877255698063, + "tokens_seen": 14090240 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021428571428571427, + "loss": 5.2005, + "theoretical_loss": 6.494565268992613, + "tokens_seen": 14155776 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002152777777777778, + "loss": 5.6155, + "theoretical_loss": 6.489284667824741, + "tokens_seen": 14221312 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021626984126984128, + "loss": 5.6027, + "theoretical_loss": 6.484035123397694, + "tokens_seen": 14286848 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021726190476190478, + "loss": 5.5959, + "theoretical_loss": 6.478816311847305, + "tokens_seen": 14352384 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021825396825396825, + "loss": 5.7842, + "theoretical_loss": 6.473627914145865, + "tokens_seen": 14417920 + }, + { + "epoch": 0.0, + "learning_rate": 0.00021924603174603176, + "loss": 5.4174, + "theoretical_loss": 6.468469616008303, + "tokens_seen": 14483456 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022023809523809523, + "loss": 5.4423, + "theoretical_loss": 6.463341107800611, + "tokens_seen": 14548992 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022123015873015873, + "loss": 5.2413, + "theoretical_loss": 6.458242084450426, + "tokens_seen": 14614528 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002222222222222222, + "loss": 5.4841, + "theoretical_loss": 6.453172245359726, + "tokens_seen": 14680064 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 20187, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.4210638999938965, + "objective/train/theoretical_loss": 6.448131294319589, + "objective/train/tokens_used": 35205600, + "theoretical_loss": 6.448131294319589, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022321428571428573, + "loss": 5.318, + "theoretical_loss": 6.448131294319589, + "tokens_seen": 14745600 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002242063492063492, + "loss": 5.5298, + "theoretical_loss": 6.443118939426935, + "tokens_seen": 14811136 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002251984126984127, + "loss": 5.5104, + "theoretical_loss": 6.438134893003209, + "tokens_seen": 14876672 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022619047619047618, + "loss": 5.4089, + "theoretical_loss": 6.43317887151496, + "tokens_seen": 14942208 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022718253968253969, + "loss": 5.5783, + "theoretical_loss": 6.428250595496234, + "tokens_seen": 15007744 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022817460317460316, + "loss": 5.4029, + "theoretical_loss": 6.4233497894727805, + "tokens_seen": 15073280 + }, + { + "epoch": 0.0, + "learning_rate": 0.00022916666666666666, + "loss": 5.6043, + "theoretical_loss": 6.418476181887964, + "tokens_seen": 15138816 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023015873015873016, + "loss": 5.5123, + "theoretical_loss": 6.413629505030386, + "tokens_seen": 15204352 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023115079365079367, + "loss": 5.5084, + "theoretical_loss": 6.408809494963135, + "tokens_seen": 15269888 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023214285714285717, + "loss": 5.269, + "theoretical_loss": 6.404015891454651, + "tokens_seen": 15335424 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023313492063492064, + "loss": 5.4379, + "theoretical_loss": 6.399248437911128, + "tokens_seen": 15400960 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023412698412698414, + "loss": 5.4346, + "theoretical_loss": 6.394506881310447, + "tokens_seen": 15466496 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023511904761904762, + "loss": 5.7191, + "theoretical_loss": 6.38979097213757, + "tokens_seen": 15532032 + }, + { + "epoch": 0.0, + "learning_rate": 0.00023611111111111112, + "loss": 5.5484, + "theoretical_loss": 6.385100464321383, + "tokens_seen": 15597568 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002371031746031746, + "loss": 5.4379, + "theoretical_loss": 6.380435115172919, + "tokens_seen": 15663104 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002380952380952381, + "loss": 5.4785, + "theoretical_loss": 6.375794685324962, + "tokens_seen": 15728640 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002390873015873016, + "loss": 5.2088, + "theoretical_loss": 6.371178938672962, + "tokens_seen": 15794176 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002400793650793651, + "loss": 5.5076, + "theoretical_loss": 6.366587642317243, + "tokens_seen": 15859712 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024107142857142857, + "loss": 5.6351, + "theoretical_loss": 6.362020566506475, + "tokens_seen": 15925248 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024206349206349207, + "loss": 5.6029, + "theoretical_loss": 6.3574774845823665, + "tokens_seen": 15990784 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024305555555555555, + "loss": 5.1969, + "theoretical_loss": 6.352958172925549, + "tokens_seen": 16056320 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024404761904761905, + "loss": 5.285, + "theoretical_loss": 6.348462410902625, + "tokens_seen": 16121856 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024503968253968255, + "loss": 5.6805, + "theoretical_loss": 6.343989980814362, + "tokens_seen": 16187392 + }, + { + "epoch": 0.0, + "learning_rate": 0.000246031746031746, + "loss": 5.5855, + "theoretical_loss": 6.339540667844965, + "tokens_seen": 16252928 + }, + { + "epoch": 0.0, + "learning_rate": 0.00024702380952380955, + "loss": 5.3559, + "theoretical_loss": 6.335114260012455, + "tokens_seen": 16318464 + }, + { + "epoch": 0.0, + "objective/train/docs_used": 20737, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.463630199432373, + "objective/train/theoretical_loss": 6.330710548120079, + "objective/train/tokens_used": 36844000, + "theoretical_loss": 6.330710548120079, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.000248015873015873, + "loss": 5.515, + "theoretical_loss": 6.330710548120079, + "tokens_seen": 16384000 + }, + { + "epoch": 0.0, + "learning_rate": 0.0002490079365079365, + "loss": 5.3596, + "theoretical_loss": 6.326329325708746, + "tokens_seen": 16449536 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025, + "loss": 5.249, + "theoretical_loss": 6.321970389010465, + "tokens_seen": 16515072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002509920634920635, + "loss": 5.3675, + "theoretical_loss": 6.317633536902759, + "tokens_seen": 16580608 + }, + { + "epoch": 0.01, + "learning_rate": 0.000251984126984127, + "loss": 5.0919, + "theoretical_loss": 6.313318570864016, + "tokens_seen": 16646144 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025297619047619046, + "loss": 5.1228, + "theoretical_loss": 6.30902529492978, + "tokens_seen": 16711680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025396825396825396, + "loss": 5.4573, + "theoretical_loss": 6.304753515649935, + "tokens_seen": 16777216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025496031746031746, + "loss": 5.1499, + "theoretical_loss": 6.30050304204677, + "tokens_seen": 16842752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025595238095238096, + "loss": 5.6432, + "theoretical_loss": 6.296273685573913, + "tokens_seen": 16908288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002569444444444444, + "loss": 5.439, + "theoretical_loss": 6.292065260076094, + "tokens_seen": 16973824 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025793650793650796, + "loss": 5.2959, + "theoretical_loss": 6.287877581749726, + "tokens_seen": 17039360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025892857142857146, + "loss": 5.3502, + "theoretical_loss": 6.2837104691042915, + "tokens_seen": 17104896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00025992063492063497, + "loss": 5.5134, + "theoretical_loss": 6.279563742924502, + "tokens_seen": 17170432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002609126984126984, + "loss": 5.4871, + "theoretical_loss": 6.275437226233224, + "tokens_seen": 17235968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002619047619047619, + "loss": 5.3132, + "theoretical_loss": 6.271330744255137, + "tokens_seen": 17301504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002628968253968254, + "loss": 5.1796, + "theoretical_loss": 6.267244124381133, + "tokens_seen": 17367040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002638888888888889, + "loss": 5.2656, + "theoretical_loss": 6.2631771961334035, + "tokens_seen": 17432576 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026488095238095237, + "loss": 5.5896, + "theoretical_loss": 6.259129791131242, + "tokens_seen": 17498112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026587301587301587, + "loss": 5.14, + "theoretical_loss": 6.255101743057493, + "tokens_seen": 17563648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026686507936507937, + "loss": 5.3606, + "theoretical_loss": 6.251092887625685, + "tokens_seen": 17629184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00026785714285714287, + "loss": 5.487, + "theoretical_loss": 6.247103062547796, + "tokens_seen": 17694720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002688492063492063, + "loss": 5.2887, + "theoretical_loss": 6.243132107502647, + "tokens_seen": 17760256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002698412698412698, + "loss": 5.4254, + "theoretical_loss": 6.239179864104911, + "tokens_seen": 17825792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002708333333333333, + "loss": 5.4668, + "theoretical_loss": 6.235246175874727, + "tokens_seen": 17891328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002718253968253968, + "loss": 5.3584, + "theoretical_loss": 6.231330888207894, + "tokens_seen": 17956864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 21782, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.119491100311279, + "objective/train/theoretical_loss": 6.227433848346639, + "objective/train/tokens_used": 38482400, + "theoretical_loss": 6.227433848346639, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002728174603174603, + "loss": 5.1069, + "theoretical_loss": 6.227433848346639, + "tokens_seen": 18022400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002738095238095238, + "loss": 5.1568, + "theoretical_loss": 6.22355490535095, + "tokens_seen": 18087936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002748015873015873, + "loss": 5.4409, + "theoretical_loss": 6.219693910070452, + "tokens_seen": 18153472 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027579365079365083, + "loss": 5.1939, + "theoretical_loss": 6.215850715116817, + "tokens_seen": 18219008 + }, + { + "epoch": 0.01, + "learning_rate": 0.00027678571428571433, + "loss": 5.2274, + "theoretical_loss": 6.212025174836697, + "tokens_seen": 18284544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002777777777777778, + "loss": 5.5299, + "theoretical_loss": 6.208217145285173, + "tokens_seen": 18350080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002787698412698413, + "loss": 5.4441, + "theoretical_loss": 6.204426484199696, + "tokens_seen": 18415616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002797619047619048, + "loss": 5.3135, + "theoretical_loss": 6.200653050974525, + "tokens_seen": 18481152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002807539682539683, + "loss": 5.189, + "theoretical_loss": 6.196896706635635, + "tokens_seen": 18546688 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028174603174603173, + "loss": 5.0786, + "theoretical_loss": 6.1931573138160925, + "tokens_seen": 18612224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028273809523809523, + "loss": 5.0434, + "theoretical_loss": 6.189434736731897, + "tokens_seen": 18677760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028373015873015873, + "loss": 5.0132, + "theoretical_loss": 6.185728841158257, + "tokens_seen": 18743296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00028472222222222223, + "loss": 5.1159, + "theoretical_loss": 6.182039494406309, + "tokens_seen": 18808832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002857142857142857, + "loss": 5.1786, + "theoretical_loss": 6.178366565300266, + "tokens_seen": 18874368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002867063492063492, + "loss": 5.1947, + "theoretical_loss": 6.174709924154978, + "tokens_seen": 18939904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002876984126984127, + "loss": 5.2396, + "theoretical_loss": 6.171069442753909, + "tokens_seen": 19005440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002886904761904762, + "loss": 5.3947, + "theoretical_loss": 6.1674449943275045, + "tokens_seen": 19070976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002896825396825397, + "loss": 4.8732, + "theoretical_loss": 6.16383645353196, + "tokens_seen": 19136512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002906746031746032, + "loss": 5.3178, + "theoretical_loss": 6.160243696428367, + "tokens_seen": 19202048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002916666666666667, + "loss": 5.1441, + "theoretical_loss": 6.156666600462238, + "tokens_seen": 19267584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002926587301587302, + "loss": 5.2695, + "theoretical_loss": 6.153105044443393, + "tokens_seen": 19333120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002936507936507937, + "loss": 5.3181, + "theoretical_loss": 6.149558908526206, + "tokens_seen": 19398656 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029464285714285714, + "loss": 5.2188, + "theoretical_loss": 6.146028074190217, + "tokens_seen": 19464192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029563492063492064, + "loss": 5.1652, + "theoretical_loss": 6.14251242422106, + "tokens_seen": 19529728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029662698412698414, + "loss": 5.3475, + "theoretical_loss": 6.139011842691756, + "tokens_seen": 19595264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 22379, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.661308765411377, + "objective/train/theoretical_loss": 6.135526214944321, + "objective/train/tokens_used": 40120800, + "theoretical_loss": 6.135526214944321, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00029761904761904765, + "loss": 5.6218, + "theoretical_loss": 6.135526214944321, + "tokens_seen": 19660800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002986111111111111, + "loss": 5.4528, + "theoretical_loss": 6.1320554275717, + "tokens_seen": 19726336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0002996031746031746, + "loss": 5.1971, + "theoretical_loss": 6.128599368400017, + "tokens_seen": 19791872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003005952380952381, + "loss": 5.2823, + "theoretical_loss": 6.125157926471134, + "tokens_seen": 19857408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003015873015873016, + "loss": 5.1154, + "theoretical_loss": 6.121730992025516, + "tokens_seen": 19922944 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030257936507936505, + "loss": 4.8293, + "theoretical_loss": 6.118318456485394, + "tokens_seen": 19988480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030357142857142855, + "loss": 5.0591, + "theoretical_loss": 6.114920212438209, + "tokens_seen": 20054016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030456349206349205, + "loss": 5.2452, + "theoretical_loss": 6.111536153620355, + "tokens_seen": 20119552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003055555555555556, + "loss": 5.1158, + "theoretical_loss": 6.108166174901191, + "tokens_seen": 20185088 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030654761904761905, + "loss": 5.0927, + "theoretical_loss": 6.104810172267331, + "tokens_seen": 20250624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030753968253968255, + "loss": 5.1097, + "theoretical_loss": 6.101468042807199, + "tokens_seen": 20316160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030853174603174605, + "loss": 5.0109, + "theoretical_loss": 6.098139684695851, + "tokens_seen": 20381696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00030952380952380956, + "loss": 5.2, + "theoretical_loss": 6.094824997180048, + "tokens_seen": 20447232 + }, + { + "epoch": 0.01, + "learning_rate": 0.000310515873015873, + "loss": 4.919, + "theoretical_loss": 6.091523880563589, + "tokens_seen": 20512768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003115079365079365, + "loss": 4.7726, + "theoretical_loss": 6.088236236192881, + "tokens_seen": 20578304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003125, + "loss": 5.1579, + "theoretical_loss": 6.0849619664427586, + "tokens_seen": 20643840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003134920634920635, + "loss": 5.2649, + "theoretical_loss": 6.0817009747025415, + "tokens_seen": 20709376 + }, + { + "epoch": 0.01, + "learning_rate": 0.000314484126984127, + "loss": 4.9338, + "theoretical_loss": 6.07845316536232, + "tokens_seen": 20774912 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031547619047619046, + "loss": 5.2449, + "theoretical_loss": 6.075218443799468, + "tokens_seen": 20840448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031646825396825396, + "loss": 5.2222, + "theoretical_loss": 6.071996716365382, + "tokens_seen": 20905984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031746031746031746, + "loss": 4.8122, + "theoretical_loss": 6.068787890372443, + "tokens_seen": 20971520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00031845238095238096, + "loss": 4.983, + "theoretical_loss": 6.0655918740811865, + "tokens_seen": 21037056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003194444444444444, + "loss": 4.9428, + "theoretical_loss": 6.062408576687682, + "tokens_seen": 21102592 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032043650793650796, + "loss": 5.0493, + "theoretical_loss": 6.059237908311129, + "tokens_seen": 21168128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032142857142857147, + "loss": 4.8675, + "theoretical_loss": 6.056079779981644, + "tokens_seen": 21233664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 23601, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.199455738067627, + "objective/train/theoretical_loss": 6.052934103628253, + "objective/train/tokens_used": 41759200, + "theoretical_loss": 6.052934103628253, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032242063492063497, + "loss": 5.2637, + "theoretical_loss": 6.052934103628253, + "tokens_seen": 21299200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003234126984126984, + "loss": 5.4325, + "theoretical_loss": 6.049800792067078, + "tokens_seen": 21364736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003244047619047619, + "loss": 4.8274, + "theoretical_loss": 6.046679758989709, + "tokens_seen": 21430272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003253968253968254, + "loss": 5.2104, + "theoretical_loss": 6.043570918951775, + "tokens_seen": 21495808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003263888888888889, + "loss": 5.1036, + "theoretical_loss": 6.040474187361681, + "tokens_seen": 21561344 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032738095238095237, + "loss": 4.8811, + "theoretical_loss": 6.037389480469548, + "tokens_seen": 21626880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032837301587301587, + "loss": 5.0504, + "theoretical_loss": 6.034316715356304, + "tokens_seen": 21692416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00032936507936507937, + "loss": 5.0898, + "theoretical_loss": 6.031255809922974, + "tokens_seen": 21757952 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033035714285714287, + "loss": 5.0121, + "theoretical_loss": 6.0282066828801195, + "tokens_seen": 21823488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003313492063492063, + "loss": 5.1363, + "theoretical_loss": 6.0251692537374595, + "tokens_seen": 21889024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003323412698412698, + "loss": 5.1503, + "theoretical_loss": 6.022143442793643, + "tokens_seen": 21954560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003333333333333333, + "loss": 5.1066, + "theoretical_loss": 6.0191291711261945, + "tokens_seen": 22020096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003343253968253968, + "loss": 5.0889, + "theoretical_loss": 6.016126360581609, + "tokens_seen": 22085632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003353174603174603, + "loss": 4.9077, + "theoretical_loss": 6.013134933765605, + "tokens_seen": 22151168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003363095238095238, + "loss": 5.2813, + "theoretical_loss": 6.010154814033528, + "tokens_seen": 22216704 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033730158730158733, + "loss": 4.6999, + "theoretical_loss": 6.007185925480902, + "tokens_seen": 22282240 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033829365079365083, + "loss": 5.0728, + "theoretical_loss": 6.0042281929341375, + "tokens_seen": 22347776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00033928571428571433, + "loss": 5.1319, + "theoretical_loss": 6.001281541941363, + "tokens_seen": 22413312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003402777777777778, + "loss": 5.0915, + "theoretical_loss": 5.998345898763421, + "tokens_seen": 22478848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003412698412698413, + "loss": 4.8912, + "theoretical_loss": 5.995421190364983, + "tokens_seen": 22544384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003422619047619048, + "loss": 5.0317, + "theoretical_loss": 5.992507344405814, + "tokens_seen": 22609920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003432539682539683, + "loss": 5.1603, + "theoretical_loss": 5.9896042892321635, + "tokens_seen": 22675456 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034424603174603173, + "loss": 4.9822, + "theoretical_loss": 5.986711953868287, + "tokens_seen": 22740992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034523809523809523, + "loss": 5.3948, + "theoretical_loss": 5.983830268008107, + "tokens_seen": 22806528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034623015873015873, + "loss": 5.222, + "theoretical_loss": 5.980959162006979, + "tokens_seen": 22872064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 24859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.439995288848877, + "objective/train/theoretical_loss": 5.978098566873603, + "objective/train/tokens_used": 43397600, + "theoretical_loss": 5.978098566873603, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00034722222222222224, + "loss": 4.9666, + "theoretical_loss": 5.978098566873603, + "tokens_seen": 22937600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003482142857142857, + "loss": 4.7873, + "theoretical_loss": 5.975248414262053, + "tokens_seen": 23003136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003492063492063492, + "loss": 5.1429, + "theoretical_loss": 5.972408636463909, + "tokens_seen": 23068672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003501984126984127, + "loss": 4.8387, + "theoretical_loss": 5.969579166400528, + "tokens_seen": 23134208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003511904761904762, + "loss": 5.2741, + "theoretical_loss": 5.966759937615427, + "tokens_seen": 23199744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003521825396825397, + "loss": 4.9993, + "theoretical_loss": 5.96395088426676, + "tokens_seen": 23265280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003531746031746032, + "loss": 4.838, + "theoretical_loss": 5.961151941119932, + "tokens_seen": 23330816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003541666666666667, + "loss": 4.8937, + "theoretical_loss": 5.95836304354031, + "tokens_seen": 23396352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003551587301587302, + "loss": 5.1755, + "theoretical_loss": 5.9555841274860395, + "tokens_seen": 23461888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003561507936507937, + "loss": 5.0192, + "theoretical_loss": 5.952815129500973, + "tokens_seen": 23527424 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035714285714285714, + "loss": 5.1528, + "theoretical_loss": 5.950055986707699, + "tokens_seen": 23592960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035813492063492064, + "loss": 5.2473, + "theoretical_loss": 5.9473066368006755, + "tokens_seen": 23658496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00035912698412698415, + "loss": 5.1431, + "theoretical_loss": 5.944567018039454, + "tokens_seen": 23724032 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036011904761904765, + "loss": 5.2078, + "theoretical_loss": 5.941837069242023, + "tokens_seen": 23789568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003611111111111111, + "loss": 5.2074, + "theoretical_loss": 5.939116729778224, + "tokens_seen": 23855104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003621031746031746, + "loss": 5.045, + "theoretical_loss": 5.936405939563279, + "tokens_seen": 23920640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003630952380952381, + "loss": 5.0272, + "theoretical_loss": 5.93370463905141, + "tokens_seen": 23986176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003640873015873016, + "loss": 5.0628, + "theoretical_loss": 5.931012769229536, + "tokens_seen": 24051712 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036507936507936505, + "loss": 4.6127, + "theoretical_loss": 5.928330271611081, + "tokens_seen": 24117248 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036607142857142855, + "loss": 4.8408, + "theoretical_loss": 5.925657088229862, + "tokens_seen": 24182784 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036706349206349205, + "loss": 4.9603, + "theoretical_loss": 5.9229931616340545, + "tokens_seen": 24248320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003680555555555556, + "loss": 5.0127, + "theoretical_loss": 5.920338434880263, + "tokens_seen": 24313856 + }, + { + "epoch": 0.01, + "learning_rate": 0.00036904761904761905, + "loss": 5.3699, + "theoretical_loss": 5.9176928515276535, + "tokens_seen": 24379392 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037003968253968255, + "loss": 4.9974, + "theoretical_loss": 5.915056355632197, + "tokens_seen": 24444928 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037103174603174606, + "loss": 4.9348, + "theoretical_loss": 5.912428891740967, + "tokens_seen": 24510464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 25411, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.414747714996338, + "objective/train/theoretical_loss": 5.90981040488653, + "objective/train/tokens_used": 45036000, + "theoretical_loss": 5.90981040488653, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037202380952380956, + "loss": 5.0376, + "theoretical_loss": 5.90981040488653, + "tokens_seen": 24576000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000373015873015873, + "loss": 5.0724, + "theoretical_loss": 5.907200840581417, + "tokens_seen": 24641536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003740079365079365, + "loss": 5.2247, + "theoretical_loss": 5.904600144812672, + "tokens_seen": 24707072 + }, + { + "epoch": 0.01, + "learning_rate": 0.000375, + "loss": 5.4811, + "theoretical_loss": 5.902008264036468, + "tokens_seen": 24772608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003759920634920635, + "loss": 4.9107, + "theoretical_loss": 5.899425145172803, + "tokens_seen": 24838144 + }, + { + "epoch": 0.01, + "learning_rate": 0.000376984126984127, + "loss": 5.0933, + "theoretical_loss": 5.896850735600281, + "tokens_seen": 24903680 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037797619047619046, + "loss": 5.1802, + "theoretical_loss": 5.8942849831509445, + "tokens_seen": 24969216 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037896825396825396, + "loss": 5.2521, + "theoretical_loss": 5.891727836105194, + "tokens_seen": 25034752 + }, + { + "epoch": 0.01, + "learning_rate": 0.00037996031746031746, + "loss": 4.8607, + "theoretical_loss": 5.889179243186776, + "tokens_seen": 25100288 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038095238095238096, + "loss": 5.0973, + "theoretical_loss": 5.886639153557828, + "tokens_seen": 25165824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003819444444444444, + "loss": 5.3397, + "theoretical_loss": 5.8841075168140105, + "tokens_seen": 25231360 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038293650793650797, + "loss": 4.7777, + "theoretical_loss": 5.88158428297969, + "tokens_seen": 25296896 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038392857142857147, + "loss": 4.7219, + "theoretical_loss": 5.879069402503189, + "tokens_seen": 25362432 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038492063492063497, + "loss": 5.0299, + "theoretical_loss": 5.876562826252119, + "tokens_seen": 25427968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003859126984126984, + "loss": 4.8369, + "theoretical_loss": 5.874064505508748, + "tokens_seen": 25493504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003869047619047619, + "loss": 5.0017, + "theoretical_loss": 5.871574391965453, + "tokens_seen": 25559040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003878968253968254, + "loss": 4.9705, + "theoretical_loss": 5.8690924377202265, + "tokens_seen": 25624576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003888888888888889, + "loss": 5.034, + "theoretical_loss": 5.866618595272241, + "tokens_seen": 25690112 + }, + { + "epoch": 0.01, + "learning_rate": 0.00038988095238095237, + "loss": 4.7181, + "theoretical_loss": 5.864152817517482, + "tokens_seen": 25755648 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039087301587301587, + "loss": 4.9562, + "theoretical_loss": 5.8616950577444245, + "tokens_seen": 25821184 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039186507936507937, + "loss": 4.9405, + "theoretical_loss": 5.859245269629783, + "tokens_seen": 25886720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003928571428571429, + "loss": 5.099, + "theoretical_loss": 5.85680340723431, + "tokens_seen": 25952256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003938492063492063, + "loss": 4.6503, + "theoretical_loss": 5.85436942499865, + "tokens_seen": 26017792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003948412698412698, + "loss": 4.8672, + "theoretical_loss": 5.851943277739253, + "tokens_seen": 26083328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003958333333333333, + "loss": 5.0201, + "theoretical_loss": 5.84952492064434, + "tokens_seen": 26148864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 26668, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 5.337137222290039, + "objective/train/theoretical_loss": 5.847114309269919, + "objective/train/tokens_used": 46674400, + "theoretical_loss": 5.847114309269919, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003968253968253968, + "loss": 5.1138, + "theoretical_loss": 5.847114309269919, + "tokens_seen": 26214400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0003978174603174603, + "loss": 4.8239, + "theoretical_loss": 5.844711399535855, + "tokens_seen": 26279936 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039880952380952383, + "loss": 4.7304, + "theoretical_loss": 5.842316147722, + "tokens_seen": 26345472 + }, + { + "epoch": 0.01, + "learning_rate": 0.00039980158730158733, + "loss": 5.0092, + "theoretical_loss": 5.839928510464356, + "tokens_seen": 26411008 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040079365079365083, + "loss": 4.4343, + "theoretical_loss": 5.837548444751306, + "tokens_seen": 26476544 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040178571428571433, + "loss": 4.9309, + "theoretical_loss": 5.835175907919885, + "tokens_seen": 26542080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004027777777777778, + "loss": 5.0148, + "theoretical_loss": 5.832810857652097, + "tokens_seen": 26607616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004037698412698413, + "loss": 4.9709, + "theoretical_loss": 5.830453251971296, + "tokens_seen": 26673152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004047619047619048, + "loss": 5.1225, + "theoretical_loss": 5.8281030492385835, + "tokens_seen": 26738688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004057539682539683, + "loss": 4.9439, + "theoretical_loss": 5.82576020814929, + "tokens_seen": 26804224 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040674603174603173, + "loss": 5.154, + "theoretical_loss": 5.823424687729469, + "tokens_seen": 26869760 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040773809523809523, + "loss": 4.6484, + "theoretical_loss": 5.821096447332456, + "tokens_seen": 26935296 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040873015873015874, + "loss": 4.6262, + "theoretical_loss": 5.818775446635469, + "tokens_seen": 27000832 + }, + { + "epoch": 0.01, + "learning_rate": 0.00040972222222222224, + "loss": 5.0236, + "theoretical_loss": 5.816461645636238, + "tokens_seen": 27066368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004107142857142857, + "loss": 5.1234, + "theoretical_loss": 5.814155004649702, + "tokens_seen": 27131904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004117063492063492, + "loss": 4.6509, + "theoretical_loss": 5.811855484304724, + "tokens_seen": 27197440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004126984126984127, + "loss": 4.9814, + "theoretical_loss": 5.809563045540864, + "tokens_seen": 27262976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004136904761904762, + "loss": 5.0187, + "theoretical_loss": 5.8072776496051866, + "tokens_seen": 27328512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004146825396825397, + "loss": 4.9255, + "theoretical_loss": 5.804999258049106, + "tokens_seen": 27394048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004156746031746032, + "loss": 5.0265, + "theoretical_loss": 5.802727832725283, + "tokens_seen": 27459584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004166666666666667, + "loss": 4.8827, + "theoretical_loss": 5.800463335784541, + "tokens_seen": 27525120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004176587301587302, + "loss": 5.0962, + "theoretical_loss": 5.798205729672842, + "tokens_seen": 27590656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004186507936507937, + "loss": 5.0409, + "theoretical_loss": 5.795954977128286, + "tokens_seen": 27656192 + }, + { + "epoch": 0.01, + "learning_rate": 0.00041964285714285714, + "loss": 4.8332, + "theoretical_loss": 5.7937110411781525, + "tokens_seen": 27721728 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042063492063492065, + "loss": 4.813, + "theoretical_loss": 5.79147388513598, + "tokens_seen": 27787264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 27295, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.60736083984375, + "objective/train/theoretical_loss": 5.789243472598683, + "objective/train/tokens_used": 48312800, + "theoretical_loss": 5.789243472598683, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042162698412698415, + "loss": 4.735, + "theoretical_loss": 5.789243472598683, + "tokens_seen": 27852800 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042261904761904765, + "loss": 5.0103, + "theoretical_loss": 5.787019767443696, + "tokens_seen": 27918336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004236111111111111, + "loss": 4.9487, + "theoretical_loss": 5.784802733826166, + "tokens_seen": 27983872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004246031746031746, + "loss": 5.0393, + "theoretical_loss": 5.782592336176171, + "tokens_seen": 28049408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004255952380952381, + "loss": 4.8239, + "theoretical_loss": 5.780388539195972, + "tokens_seen": 28114944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004265873015873016, + "loss": 4.8301, + "theoretical_loss": 5.778191307857307, + "tokens_seen": 28180480 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042757936507936505, + "loss": 4.8818, + "theoretical_loss": 5.776000607398707, + "tokens_seen": 28246016 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042857142857142855, + "loss": 5.0365, + "theoretical_loss": 5.773816403322854, + "tokens_seen": 28311552 + }, + { + "epoch": 0.01, + "learning_rate": 0.00042956349206349205, + "loss": 4.8806, + "theoretical_loss": 5.7716386613939665, + "tokens_seen": 28377088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004305555555555556, + "loss": 4.8212, + "theoretical_loss": 5.769467347635221, + "tokens_seen": 28442624 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043154761904761905, + "loss": 4.6856, + "theoretical_loss": 5.767302428326196, + "tokens_seen": 28508160 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043253968253968256, + "loss": 5.1099, + "theoretical_loss": 5.765143870000358, + "tokens_seen": 28573696 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043353174603174606, + "loss": 4.9942, + "theoretical_loss": 5.762991639442574, + "tokens_seen": 28639232 + }, + { + "epoch": 0.01, + "learning_rate": 0.00043452380952380956, + "loss": 4.7269, + "theoretical_loss": 5.7608457036866465, + "tokens_seen": 28704768 + }, + { + "epoch": 0.01, + "learning_rate": 0.000435515873015873, + "loss": 4.7079, + "theoretical_loss": 5.758706030012889, + "tokens_seen": 28770304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004365079365079365, + "loss": 5.0535, + "theoretical_loss": 5.756572585945728, + "tokens_seen": 28835840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004375, + "loss": 4.9953, + "theoretical_loss": 5.754445339251326, + "tokens_seen": 28901376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004384920634920635, + "loss": 4.8239, + "theoretical_loss": 5.752324257935244, + "tokens_seen": 28966912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000439484126984127, + "loss": 4.8738, + "theoretical_loss": 5.750209310240125, + "tokens_seen": 29032448 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044047619047619046, + "loss": 4.5774, + "theoretical_loss": 5.7481004646434055, + "tokens_seen": 29097984 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044146825396825396, + "loss": 4.7821, + "theoretical_loss": 5.745997689855058, + "tokens_seen": 29163520 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044246031746031746, + "loss": 4.9197, + "theoretical_loss": 5.743900954815356, + "tokens_seen": 29229056 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044345238095238096, + "loss": 4.9178, + "theoretical_loss": 5.741810228692663, + "tokens_seen": 29294592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004444444444444444, + "loss": 4.9811, + "theoretical_loss": 5.739725480881262, + "tokens_seen": 29360128 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044543650793650797, + "loss": 4.595, + "theoretical_loss": 5.737646680999193, + "tokens_seen": 29425664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 28553, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.193212032318115, + "objective/train/theoretical_loss": 5.73557379888612, + "objective/train/tokens_used": 49951200, + "theoretical_loss": 5.73557379888612, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044642857142857147, + "loss": 4.6289, + "theoretical_loss": 5.73557379888612, + "tokens_seen": 29491200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00044742063492063497, + "loss": 4.5175, + "theoretical_loss": 5.733506804601236, + "tokens_seen": 29556736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004484126984126984, + "loss": 4.97, + "theoretical_loss": 5.7314456684211725, + "tokens_seen": 29622272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004494047619047619, + "loss": 4.9007, + "theoretical_loss": 5.729390360837952, + "tokens_seen": 29687808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004503968253968254, + "loss": 5.0718, + "theoretical_loss": 5.7273408525569485, + "tokens_seen": 29753344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004513888888888889, + "loss": 4.8076, + "theoretical_loss": 5.725297114494884, + "tokens_seen": 29818880 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045238095238095237, + "loss": 4.9151, + "theoretical_loss": 5.7232591177778405, + "tokens_seen": 29884416 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045337301587301587, + "loss": 4.7611, + "theoretical_loss": 5.7212268337393, + "tokens_seen": 29949952 + }, + { + "epoch": 0.01, + "learning_rate": 0.00045436507936507937, + "loss": 4.6533, + "theoretical_loss": 5.719200233918203, + "tokens_seen": 30015488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004553571428571429, + "loss": 4.7689, + "theoretical_loss": 5.717179290057032, + "tokens_seen": 30081024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004563492063492063, + "loss": 4.753, + "theoretical_loss": 5.715163974099917, + "tokens_seen": 30146560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004573412698412698, + "loss": 4.9783, + "theoretical_loss": 5.713154258190757, + "tokens_seen": 30212096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004583333333333333, + "loss": 4.8919, + "theoretical_loss": 5.711150114671375, + "tokens_seen": 30277632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004593253968253968, + "loss": 4.6246, + "theoretical_loss": 5.709151516079683, + "tokens_seen": 30343168 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046031746031746033, + "loss": 4.8047, + "theoretical_loss": 5.707158435147875, + "tokens_seen": 30408704 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046130952380952383, + "loss": 4.7619, + "theoretical_loss": 5.705170844800628, + "tokens_seen": 30474240 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046230158730158733, + "loss": 5.0389, + "theoretical_loss": 5.703188718153347, + "tokens_seen": 30539776 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046329365079365083, + "loss": 4.7978, + "theoretical_loss": 5.7012120285104055, + "tokens_seen": 30605312 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046428571428571433, + "loss": 4.8182, + "theoretical_loss": 5.699240749363417, + "tokens_seen": 30670848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004652777777777778, + "loss": 5.1046, + "theoretical_loss": 5.69727485438953, + "tokens_seen": 30736384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004662698412698413, + "loss": 4.9301, + "theoretical_loss": 5.695314317449732, + "tokens_seen": 30801920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004672619047619048, + "loss": 4.9513, + "theoretical_loss": 5.69335911258718, + "tokens_seen": 30867456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004682539682539683, + "loss": 4.7506, + "theoretical_loss": 5.691409214025544, + "tokens_seen": 30932992 + }, + { + "epoch": 0.01, + "learning_rate": 0.00046924603174603173, + "loss": 4.9203, + "theoretical_loss": 5.689464596167383, + "tokens_seen": 30998528 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047023809523809523, + "loss": 4.6588, + "theoretical_loss": 5.687525233592513, + "tokens_seen": 31064064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 29210, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.980744361877441, + "objective/train/theoretical_loss": 5.685591101056422, + "objective/train/tokens_used": 51589600, + "theoretical_loss": 5.685591101056422, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047123015873015874, + "loss": 4.8131, + "theoretical_loss": 5.685591101056422, + "tokens_seen": 31129600 + }, + { + "epoch": 0.01, + "learning_rate": 0.00047222222222222224, + "loss": 4.9729, + "theoretical_loss": 5.683662173488678, + "tokens_seen": 31195136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004732142857142857, + "loss": 4.9988, + "theoretical_loss": 5.681738425991377, + "tokens_seen": 31260672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004742063492063492, + "loss": 4.9296, + "theoretical_loss": 5.679819833837586, + "tokens_seen": 31326208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004751984126984127, + "loss": 4.8866, + "theoretical_loss": 5.677906372469826, + "tokens_seen": 31391744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004761904761904762, + "loss": 4.8619, + "theoretical_loss": 5.675998017498549, + "tokens_seen": 31457280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004771825396825397, + "loss": 4.4881, + "theoretical_loss": 5.674094744700648, + "tokens_seen": 31522816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004781746031746032, + "loss": 4.7385, + "theoretical_loss": 5.672196530017979, + "tokens_seen": 31588352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004791666666666667, + "loss": 4.9446, + "theoretical_loss": 5.670303349555893, + "tokens_seen": 31653888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004801587301587302, + "loss": 4.5805, + "theoretical_loss": 5.668415179581795, + "tokens_seen": 31719424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004811507936507937, + "loss": 4.943, + "theoretical_loss": 5.666531996523711, + "tokens_seen": 31784960 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048214285714285715, + "loss": 4.9994, + "theoretical_loss": 5.664653776968867, + "tokens_seen": 31850496 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048313492063492065, + "loss": 4.9432, + "theoretical_loss": 5.6627804976622995, + "tokens_seen": 31916032 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048412698412698415, + "loss": 4.4612, + "theoretical_loss": 5.660912135505461, + "tokens_seen": 31981568 + }, + { + "epoch": 0.01, + "learning_rate": 0.00048511904761904765, + "loss": 5.0296, + "theoretical_loss": 5.659048667554853, + "tokens_seen": 32047104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004861111111111111, + "loss": 4.7342, + "theoretical_loss": 5.657190071020672, + "tokens_seen": 32112640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004871031746031746, + "loss": 5.0271, + "theoretical_loss": 5.65533632326547, + "tokens_seen": 32178176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004880952380952381, + "loss": 4.5208, + "theoretical_loss": 5.6534874018028205, + "tokens_seen": 32243712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004890873015873016, + "loss": 4.8883, + "theoretical_loss": 5.651643284296018, + "tokens_seen": 32309248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004900793650793651, + "loss": 4.5918, + "theoretical_loss": 5.6498039485567695, + "tokens_seen": 32374784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004910714285714286, + "loss": 4.8362, + "theoretical_loss": 5.6479693725439155, + "tokens_seen": 32440320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000492063492063492, + "loss": 4.4816, + "theoretical_loss": 5.646139534362161, + "tokens_seen": 32505856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004930555555555556, + "loss": 4.7304, + "theoretical_loss": 5.644314412260815, + "tokens_seen": 32571392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004940476190476191, + "loss": 4.7639, + "theoretical_loss": 5.642493984632544, + "tokens_seen": 32636928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004950396825396826, + "loss": 4.486, + "theoretical_loss": 5.640678230012151, + "tokens_seen": 32702464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 30192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.205135822296143, + "objective/train/theoretical_loss": 5.638867127075349, + "objective/train/tokens_used": 53228000, + "theoretical_loss": 5.638867127075349, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.000496031746031746, + "loss": 4.6383, + "theoretical_loss": 5.638867127075349, + "tokens_seen": 32768000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004970238095238095, + "loss": 4.7665, + "theoretical_loss": 5.63706065463756, + "tokens_seen": 32833536 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498015873015873, + "loss": 4.6651, + "theoretical_loss": 5.635258791652723, + "tokens_seen": 32899072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990079365079365, + "loss": 4.6032, + "theoretical_loss": 5.633461517212115, + "tokens_seen": 32964608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0005, + "loss": 4.8611, + "theoretical_loss": 5.631668810543181, + "tokens_seen": 33030144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999899699097292, + "loss": 4.4403, + "theoretical_loss": 5.629880651008384, + "tokens_seen": 33095680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999799398194584, + "loss": 4.6342, + "theoretical_loss": 5.628097018104057, + "tokens_seen": 33161216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999699097291876, + "loss": 4.789, + "theoretical_loss": 5.626317891459275, + "tokens_seen": 33226752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999598796389167, + "loss": 4.4669, + "theoretical_loss": 5.6245432508347335, + "tokens_seen": 33292288 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499949849548646, + "loss": 4.5265, + "theoretical_loss": 5.622773076121648, + "tokens_seen": 33357824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999398194583751, + "loss": 4.8668, + "theoretical_loss": 5.621007347340646, + "tokens_seen": 33423360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999297893681044, + "loss": 4.663, + "theoretical_loss": 5.619246044640693, + "tokens_seen": 33488896 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999197592778335, + "loss": 4.51, + "theoretical_loss": 5.617489148298015, + "tokens_seen": 33554432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004999097291875627, + "loss": 4.4827, + "theoretical_loss": 5.615736638715033, + "tokens_seen": 33619968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998996990972919, + "loss": 4.6552, + "theoretical_loss": 5.613988496419315, + "tokens_seen": 33685504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998896690070211, + "loss": 4.5444, + "theoretical_loss": 5.6122447020625374, + "tokens_seen": 33751040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998796389167503, + "loss": 4.7462, + "theoretical_loss": 5.610505236419448, + "tokens_seen": 33816576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998696088264795, + "loss": 4.6343, + "theoretical_loss": 5.608770080386854, + "tokens_seen": 33882112 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998595787362087, + "loss": 4.3194, + "theoretical_loss": 5.6070392149826045, + "tokens_seen": 33947648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998495486459378, + "loss": 4.7376, + "theoretical_loss": 5.605312621344602, + "tokens_seen": 34013184 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499839518555667, + "loss": 4.6319, + "theoretical_loss": 5.603590280729803, + "tokens_seen": 34078720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998294884653962, + "loss": 4.3936, + "theoretical_loss": 5.601872174513245, + "tokens_seen": 34144256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998194583751254, + "loss": 4.8045, + "theoretical_loss": 5.60015828418708, + "tokens_seen": 34209792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004998094282848546, + "loss": 4.8346, + "theoretical_loss": 5.59844859135961, + "tokens_seen": 34275328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997993981945837, + "loss": 4.4444, + "theoretical_loss": 5.596743077754337, + "tokens_seen": 34340864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 30758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.619051933288574, + "objective/train/theoretical_loss": 5.5950417252090325, + "objective/train/tokens_used": 54866400, + "theoretical_loss": 5.5950417252090325, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997893681043129, + "loss": 4.5684, + "theoretical_loss": 5.5950417252090325, + "tokens_seen": 34406400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997793380140421, + "loss": 4.7403, + "theoretical_loss": 5.593344515674795, + "tokens_seen": 34471936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997693079237714, + "loss": 4.7682, + "theoretical_loss": 5.591651431215139, + "tokens_seen": 34537472 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997592778335005, + "loss": 4.9194, + "theoretical_loss": 5.589962454005072, + "tokens_seen": 34603008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997492477432298, + "loss": 4.7926, + "theoretical_loss": 5.588277566330207, + "tokens_seen": 34668544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997392176529588, + "loss": 4.5919, + "theoretical_loss": 5.586596750585855, + "tokens_seen": 34734080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997291875626881, + "loss": 4.8471, + "theoretical_loss": 5.584919989276146, + "tokens_seen": 34799616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997191574724173, + "loss": 4.8823, + "theoretical_loss": 5.583247265013155, + "tokens_seen": 34865152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004997091273821465, + "loss": 4.7952, + "theoretical_loss": 5.581578560516027, + "tokens_seen": 34930688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996990972918757, + "loss": 4.8327, + "theoretical_loss": 5.57991385861013, + "tokens_seen": 34996224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996890672016048, + "loss": 4.3745, + "theoretical_loss": 5.578253142226191, + "tokens_seen": 35061760 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499679037111334, + "loss": 4.6921, + "theoretical_loss": 5.576596394399467, + "tokens_seen": 35127296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996690070210632, + "loss": 4.6698, + "theoretical_loss": 5.574943598268902, + "tokens_seen": 35192832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996589769307924, + "loss": 4.9374, + "theoretical_loss": 5.573294737076306, + "tokens_seen": 35258368 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996489468405216, + "loss": 4.5814, + "theoretical_loss": 5.571649794165537, + "tokens_seen": 35323904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996389167502507, + "loss": 4.6064, + "theoretical_loss": 5.570008752981688, + "tokens_seen": 35389440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996288866599799, + "loss": 4.7137, + "theoretical_loss": 5.5683715970702945, + "tokens_seen": 35454976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996188565697091, + "loss": 4.7919, + "theoretical_loss": 5.566738310076531, + "tokens_seen": 35520512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004996088264794383, + "loss": 4.5989, + "theoretical_loss": 5.5651088757444285, + "tokens_seen": 35586048 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995987963891675, + "loss": 4.4273, + "theoretical_loss": 5.5634832779161, + "tokens_seen": 35651584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995887662988968, + "loss": 4.6147, + "theoretical_loss": 5.561861500530965, + "tokens_seen": 35717120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995787362086258, + "loss": 4.5803, + "theoretical_loss": 5.560243527624989, + "tokens_seen": 35782656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995687061183551, + "loss": 4.762, + "theoretical_loss": 5.558629343329924, + "tokens_seen": 35848192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995586760280842, + "loss": 4.7117, + "theoretical_loss": 5.557018931872566, + "tokens_seen": 35913728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995486459378135, + "loss": 4.7915, + "theoretical_loss": 5.555412277574007, + "tokens_seen": 35979264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 31707, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.547565460205078, + "objective/train/theoretical_loss": 5.553809364848902, + "objective/train/tokens_used": 56504800, + "theoretical_loss": 5.553809364848902, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995386158475427, + "loss": 4.6425, + "theoretical_loss": 5.553809364848902, + "tokens_seen": 36044800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995285857572718, + "loss": 4.5963, + "theoretical_loss": 5.552210178204751, + "tokens_seen": 36110336 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499518555667001, + "loss": 4.6793, + "theoretical_loss": 5.550614702241162, + "tokens_seen": 36175872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004995085255767302, + "loss": 4.7561, + "theoretical_loss": 5.549022921649151, + "tokens_seen": 36241408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994984954864594, + "loss": 4.6607, + "theoretical_loss": 5.54743482121043, + "tokens_seen": 36306944 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994884653961886, + "loss": 4.6252, + "theoretical_loss": 5.545850385796712, + "tokens_seen": 36372480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994784353059178, + "loss": 4.4331, + "theoretical_loss": 5.544269600369009, + "tokens_seen": 36438016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994684052156469, + "loss": 4.643, + "theoretical_loss": 5.5426924499769505, + "tokens_seen": 36503552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994583751253761, + "loss": 4.7548, + "theoretical_loss": 5.541118919758107, + "tokens_seen": 36569088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994483450351053, + "loss": 4.7337, + "theoretical_loss": 5.539548994937308, + "tokens_seen": 36634624 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994383149448345, + "loss": 4.3641, + "theoretical_loss": 5.537982660825981, + "tokens_seen": 36700160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994282848545637, + "loss": 4.5599, + "theoretical_loss": 5.536419902821491, + "tokens_seen": 36765696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994182547642928, + "loss": 4.6875, + "theoretical_loss": 5.534860706406477, + "tokens_seen": 36831232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004994082246740221, + "loss": 4.7352, + "theoretical_loss": 5.533305057148215, + "tokens_seen": 36896768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993981945837512, + "loss": 4.6149, + "theoretical_loss": 5.531752940697968, + "tokens_seen": 36962304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993881644934805, + "loss": 4.5067, + "theoretical_loss": 5.5302043427903484, + "tokens_seen": 37027840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993781344032096, + "loss": 4.9474, + "theoretical_loss": 5.528659249242693, + "tokens_seen": 37093376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993681043129389, + "loss": 4.4017, + "theoretical_loss": 5.527117645954435, + "tokens_seen": 37158912 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499358074222668, + "loss": 4.5323, + "theoretical_loss": 5.525579518906484, + "tokens_seen": 37224448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993480441323972, + "loss": 4.4802, + "theoretical_loss": 5.524044854160613, + "tokens_seen": 37289984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993380140421264, + "loss": 4.4531, + "theoretical_loss": 5.522513637858855, + "tokens_seen": 37355520 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993279839518556, + "loss": 4.6216, + "theoretical_loss": 5.520985856222893, + "tokens_seen": 37421056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993179538615848, + "loss": 4.5407, + "theoretical_loss": 5.519461495553473, + "tokens_seen": 37486592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004993079237713139, + "loss": 4.4097, + "theoretical_loss": 5.5179405422298045, + "tokens_seen": 37552128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992978936810431, + "loss": 4.406, + "theoretical_loss": 5.516422982708983, + "tokens_seen": 37617664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 32203, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.049692630767822, + "objective/train/theoretical_loss": 5.514908803525402, + "objective/train/tokens_used": 58143200, + "theoretical_loss": 5.514908803525402, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992878635907723, + "loss": 4.3638, + "theoretical_loss": 5.514908803525402, + "tokens_seen": 37683200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992778335005015, + "loss": 4.6329, + "theoretical_loss": 5.513397991290185, + "tokens_seen": 37748736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992678034102307, + "loss": 4.6538, + "theoretical_loss": 5.511890532690611, + "tokens_seen": 37814272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992577733199598, + "loss": 4.7513, + "theoretical_loss": 5.510386414489556, + "tokens_seen": 37879808 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499247743229689, + "loss": 4.6591, + "theoretical_loss": 5.5088856235249235, + "tokens_seen": 37945344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992377131394183, + "loss": 4.4618, + "theoretical_loss": 5.507388146709105, + "tokens_seen": 38010880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992276830491475, + "loss": 4.4231, + "theoretical_loss": 5.505893971028418, + "tokens_seen": 38076416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992176529588767, + "loss": 4.2713, + "theoretical_loss": 5.504403083542568, + "tokens_seen": 38141952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004992076228686059, + "loss": 4.4092, + "theoretical_loss": 5.502915471384111, + "tokens_seen": 38207488 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499197592778335, + "loss": 4.7235, + "theoretical_loss": 5.501431121757914, + "tokens_seen": 38273024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991875626880642, + "loss": 4.3762, + "theoretical_loss": 5.49995002194063, + "tokens_seen": 38338560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991775325977934, + "loss": 4.6923, + "theoretical_loss": 5.498472159280171, + "tokens_seen": 38404096 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991675025075226, + "loss": 4.8392, + "theoretical_loss": 5.4969975211951905, + "tokens_seen": 38469632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991574724172518, + "loss": 4.5345, + "theoretical_loss": 5.495526095174563, + "tokens_seen": 38535168 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499147442326981, + "loss": 4.6407, + "theoretical_loss": 5.494057868776883, + "tokens_seen": 38600704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991374122367101, + "loss": 4.5119, + "theoretical_loss": 5.492592829629946, + "tokens_seen": 38666240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991273821464393, + "loss": 4.6456, + "theoretical_loss": 5.491130965430262, + "tokens_seen": 38731776 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991173520561685, + "loss": 4.2981, + "theoretical_loss": 5.489672263942543, + "tokens_seen": 38797312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004991073219658977, + "loss": 4.7488, + "theoretical_loss": 5.488216712999222, + "tokens_seen": 38862848 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499097291875627, + "loss": 4.5654, + "theoretical_loss": 5.486764300499962, + "tokens_seen": 38928384 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499087261785356, + "loss": 4.3462, + "theoretical_loss": 5.4853150144111655, + "tokens_seen": 38993920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990772316950853, + "loss": 4.6389, + "theoretical_loss": 5.4838688427655065, + "tokens_seen": 39059456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990672016048144, + "loss": 4.5166, + "theoretical_loss": 5.482425773661445, + "tokens_seen": 39124992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990571715145437, + "loss": 4.5858, + "theoretical_loss": 5.480985795262759, + "tokens_seen": 39190528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990471414242729, + "loss": 4.6098, + "theoretical_loss": 5.479548895798082, + "tokens_seen": 39256064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 33134, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.761120319366455, + "objective/train/theoretical_loss": 5.478115063560433, + "objective/train/tokens_used": 59781600, + "theoretical_loss": 5.478115063560433, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.000499037111334002, + "loss": 4.4957, + "theoretical_loss": 5.478115063560433, + "tokens_seen": 39321600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990270812437312, + "loss": 4.7046, + "theoretical_loss": 5.476684286906764, + "tokens_seen": 39387136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990170511534604, + "loss": 4.2948, + "theoretical_loss": 5.475256554257497, + "tokens_seen": 39452672 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004990070210631896, + "loss": 4.5969, + "theoretical_loss": 5.473831854096085, + "tokens_seen": 39518208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989969909729188, + "loss": 4.5511, + "theoretical_loss": 5.472410174968553, + "tokens_seen": 39583744 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498986960882648, + "loss": 4.4106, + "theoretical_loss": 5.470991505483065, + "tokens_seen": 39649280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989769307923771, + "loss": 4.623, + "theoretical_loss": 5.469575834309477, + "tokens_seen": 39714816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989669007021063, + "loss": 4.5077, + "theoretical_loss": 5.468163150178907, + "tokens_seen": 39780352 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989568706118355, + "loss": 4.475, + "theoretical_loss": 5.466753441883302, + "tokens_seen": 39845888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989468405215647, + "loss": 4.6451, + "theoretical_loss": 5.465346698275008, + "tokens_seen": 39911424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989368104312939, + "loss": 4.3366, + "theoretical_loss": 5.46394290826635, + "tokens_seen": 39976960 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498926780341023, + "loss": 4.4816, + "theoretical_loss": 5.462542060829209, + "tokens_seen": 40042496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989167502507523, + "loss": 4.5005, + "theoretical_loss": 5.461144144994606, + "tokens_seen": 40108032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004989067201604814, + "loss": 4.5075, + "theoretical_loss": 5.459749149852287, + "tokens_seen": 40173568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988966900702107, + "loss": 4.555, + "theoretical_loss": 5.4583570645503166, + "tokens_seen": 40239104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988866599799398, + "loss": 4.411, + "theoretical_loss": 5.45696787829467, + "tokens_seen": 40304640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988766298896691, + "loss": 4.479, + "theoretical_loss": 5.455581580348831, + "tokens_seen": 40370176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988665997993982, + "loss": 4.424, + "theoretical_loss": 5.454198160033391, + "tokens_seen": 40435712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988565697091274, + "loss": 4.6216, + "theoretical_loss": 5.452817606725657, + "tokens_seen": 40501248 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988465396188566, + "loss": 4.5915, + "theoretical_loss": 5.451439909859255, + "tokens_seen": 40566784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988365095285858, + "loss": 4.4225, + "theoretical_loss": 5.450065058923747, + "tokens_seen": 40632320 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498826479438315, + "loss": 4.4659, + "theoretical_loss": 5.448693043464235, + "tokens_seen": 40697856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988164493480441, + "loss": 4.7231, + "theoretical_loss": 5.447323853080991, + "tokens_seen": 40763392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004988064192577733, + "loss": 4.561, + "theoretical_loss": 5.445957477429067, + "tokens_seen": 40828928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987963891675025, + "loss": 4.4559, + "theoretical_loss": 5.444593906217927, + "tokens_seen": 40894464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 33820, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.709114074707031, + "objective/train/theoretical_loss": 5.443233129211069, + "objective/train/tokens_used": 61420000, + "theoretical_loss": 5.443233129211069, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987863590772317, + "loss": 4.5353, + "theoretical_loss": 5.443233129211069, + "tokens_seen": 40960000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987763289869609, + "loss": 4.5888, + "theoretical_loss": 5.441875136225658, + "tokens_seen": 41025536 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049876629889669, + "loss": 4.6609, + "theoretical_loss": 5.440519917132159, + "tokens_seen": 41091072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987562688064192, + "loss": 4.6782, + "theoretical_loss": 5.439167461853971, + "tokens_seen": 41156608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987462387161484, + "loss": 4.3218, + "theoretical_loss": 5.437817760367073, + "tokens_seen": 41222144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987362086258777, + "loss": 4.6102, + "theoretical_loss": 5.436470802699661, + "tokens_seen": 41287680 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987261785356068, + "loss": 4.4044, + "theoretical_loss": 5.43512657893179, + "tokens_seen": 41353216 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987161484453361, + "loss": 4.5902, + "theoretical_loss": 5.433785079195035, + "tokens_seen": 41418752 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004987061183550651, + "loss": 4.4395, + "theoretical_loss": 5.432446293672129, + "tokens_seen": 41484288 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986960882647944, + "loss": 4.6268, + "theoretical_loss": 5.43111021259663, + "tokens_seen": 41549824 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986860581745236, + "loss": 4.7247, + "theoretical_loss": 5.429776826252565, + "tokens_seen": 41615360 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986760280842528, + "loss": 4.7844, + "theoretical_loss": 5.428446124974103, + "tokens_seen": 41680896 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498665997993982, + "loss": 4.748, + "theoretical_loss": 5.427118099145207, + "tokens_seen": 41746432 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986559679037111, + "loss": 4.4521, + "theoretical_loss": 5.425792739199306, + "tokens_seen": 41811968 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986459378134403, + "loss": 4.5162, + "theoretical_loss": 5.424470035618963, + "tokens_seen": 41877504 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986359077231695, + "loss": 4.5787, + "theoretical_loss": 5.423149978935547, + "tokens_seen": 41943040 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986258776328987, + "loss": 4.4447, + "theoretical_loss": 5.421832559728902, + "tokens_seen": 42008576 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004986158475426279, + "loss": 4.5436, + "theoretical_loss": 5.420517768627027, + "tokens_seen": 42074112 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498605817452357, + "loss": 4.5508, + "theoretical_loss": 5.419205596305758, + "tokens_seen": 42139648 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985957873620862, + "loss": 4.3851, + "theoretical_loss": 5.417896033488446, + "tokens_seen": 42205184 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985857572718154, + "loss": 4.4232, + "theoretical_loss": 5.416589070945641, + "tokens_seen": 42270720 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985757271815446, + "loss": 4.4672, + "theoretical_loss": 5.415284699494782, + "tokens_seen": 42336256 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985656970912738, + "loss": 4.3494, + "theoretical_loss": 5.413982909999887, + "tokens_seen": 42401792 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985556670010031, + "loss": 4.246, + "theoretical_loss": 5.412683693371241, + "tokens_seen": 42467328 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985456369107321, + "loss": 4.5265, + "theoretical_loss": 5.411387040565092, + "tokens_seen": 42532864 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 34938, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.999885082244873, + "objective/train/theoretical_loss": 5.410092942583357, + "objective/train/tokens_used": 63058400, + "theoretical_loss": 5.410092942583357, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985356068204614, + "loss": 4.5919, + "theoretical_loss": 5.410092942583357, + "tokens_seen": 42598400 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985255767301905, + "loss": 4.603, + "theoretical_loss": 5.408801390473304, + "tokens_seen": 42663936 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004985155466399198, + "loss": 4.546, + "theoretical_loss": 5.407512375327272, + "tokens_seen": 42729472 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498505516549649, + "loss": 4.5326, + "theoretical_loss": 5.406225888282363, + "tokens_seen": 42795008 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984954864593782, + "loss": 4.3889, + "theoretical_loss": 5.404941920520156, + "tokens_seen": 42860544 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984854563691073, + "loss": 4.1785, + "theoretical_loss": 5.403660463266413, + "tokens_seen": 42926080 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984754262788365, + "loss": 4.5382, + "theoretical_loss": 5.402381507790792, + "tokens_seen": 42991616 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984653961885657, + "loss": 4.5693, + "theoretical_loss": 5.401105045406557, + "tokens_seen": 43057152 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984553660982949, + "loss": 4.4403, + "theoretical_loss": 5.399831067470302, + "tokens_seen": 43122688 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984453360080241, + "loss": 4.7214, + "theoretical_loss": 5.398559565381662, + "tokens_seen": 43188224 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984353059177532, + "loss": 4.4442, + "theoretical_loss": 5.397290530583037, + "tokens_seen": 43253760 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984252758274825, + "loss": 4.309, + "theoretical_loss": 5.396023954559315, + "tokens_seen": 43319296 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984152457372116, + "loss": 4.3474, + "theoretical_loss": 5.394759828837596, + "tokens_seen": 43384832 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004984052156469409, + "loss": 4.5364, + "theoretical_loss": 5.393498144986921, + "tokens_seen": 43450368 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049839518555667, + "loss": 4.647, + "theoretical_loss": 5.392238894618, + "tokens_seen": 43515904 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983851554663993, + "loss": 4.4688, + "theoretical_loss": 5.390982069382944, + "tokens_seen": 43581440 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983751253761284, + "loss": 4.555, + "theoretical_loss": 5.389727660974998, + "tokens_seen": 43646976 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983650952858576, + "loss": 4.2673, + "theoretical_loss": 5.38847566112828, + "tokens_seen": 43712512 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983550651955868, + "loss": 4.223, + "theoretical_loss": 5.387226061617516, + "tokens_seen": 43778048 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498345035105316, + "loss": 4.5944, + "theoretical_loss": 5.38597885425778, + "tokens_seen": 43843584 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983350050150452, + "loss": 4.2395, + "theoretical_loss": 5.384734030904238, + "tokens_seen": 43909120 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983249749247743, + "loss": 4.2047, + "theoretical_loss": 5.383491583451892, + "tokens_seen": 43974656 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983149448345035, + "loss": 4.6784, + "theoretical_loss": 5.382251503835324, + "tokens_seen": 44040192 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004983049147442327, + "loss": 4.5094, + "theoretical_loss": 5.381013784028445, + "tokens_seen": 44105728 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982948846539619, + "loss": 4.2719, + "theoretical_loss": 5.379778416044247, + "tokens_seen": 44171264 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 35534, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.2067718505859375, + "objective/train/theoretical_loss": 5.378545391934553, + "objective/train/tokens_used": 64696800, + "theoretical_loss": 5.378545391934553, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982848545636911, + "loss": 4.3011, + "theoretical_loss": 5.378545391934553, + "tokens_seen": 44236800 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982748244734202, + "loss": 4.3593, + "theoretical_loss": 5.377314703789773, + "tokens_seen": 44302336 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982647943831494, + "loss": 4.5546, + "theoretical_loss": 5.376086343738656, + "tokens_seen": 44367872 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982547642928786, + "loss": 4.2113, + "theoretical_loss": 5.3748603039480525, + "tokens_seen": 44433408 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982447342026079, + "loss": 4.6548, + "theoretical_loss": 5.373636576622671, + "tokens_seen": 44498944 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498234704112337, + "loss": 4.6067, + "theoretical_loss": 5.372415154004842, + "tokens_seen": 44564480 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982246740220663, + "loss": 4.5434, + "theoretical_loss": 5.371196028374278, + "tokens_seen": 44630016 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982146439317953, + "loss": 4.219, + "theoretical_loss": 5.369979192047846, + "tokens_seen": 44695552 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004982046138415246, + "loss": 4.4616, + "theoretical_loss": 5.368764637379322, + "tokens_seen": 44761088 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981945837512538, + "loss": 4.3927, + "theoretical_loss": 5.367552356759175, + "tokens_seen": 44826624 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498184553660983, + "loss": 4.3436, + "theoretical_loss": 5.366342342614322, + "tokens_seen": 44892160 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981745235707122, + "loss": 4.3641, + "theoretical_loss": 5.365134587407918, + "tokens_seen": 44957696 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981644934804413, + "loss": 4.3878, + "theoretical_loss": 5.363929083639112, + "tokens_seen": 45023232 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981544633901705, + "loss": 4.252, + "theoretical_loss": 5.362725823842838, + "tokens_seen": 45088768 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981444332998997, + "loss": 4.1657, + "theoretical_loss": 5.361524800589581, + "tokens_seen": 45154304 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981344032096289, + "loss": 4.0417, + "theoretical_loss": 5.360326006485165, + "tokens_seen": 45219840 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981243731193581, + "loss": 4.6023, + "theoretical_loss": 5.359129434170529, + "tokens_seen": 45285376 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981143430290873, + "loss": 4.3602, + "theoretical_loss": 5.357935076321514, + "tokens_seen": 45350912 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004981043129388164, + "loss": 4.4446, + "theoretical_loss": 5.35674292564864, + "tokens_seen": 45416448 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980942828485456, + "loss": 4.4122, + "theoretical_loss": 5.3555529748969, + "tokens_seen": 45481984 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980842527582748, + "loss": 4.465, + "theoretical_loss": 5.354365216845546, + "tokens_seen": 45547520 + }, + { + "epoch": 0.01, + "learning_rate": 0.000498074222668004, + "loss": 4.1501, + "theoretical_loss": 5.353179644307868, + "tokens_seen": 45613056 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980641925777333, + "loss": 4.6183, + "theoretical_loss": 5.351996250131005, + "tokens_seen": 45678592 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980541624874623, + "loss": 4.4826, + "theoretical_loss": 5.350815027195717, + "tokens_seen": 45744128 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980441323971916, + "loss": 4.0969, + "theoretical_loss": 5.349635968416189, + "tokens_seen": 45809664 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 36791, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.983624219894409, + "objective/train/theoretical_loss": 5.34845906673983, + "objective/train/tokens_used": 66335200, + "theoretical_loss": 5.34845906673983, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980341023069207, + "loss": 4.1114, + "theoretical_loss": 5.34845906673983, + "tokens_seen": 45875200 + }, + { + "epoch": 0.01, + "learning_rate": 0.00049802407221665, + "loss": 4.3748, + "theoretical_loss": 5.347284315147059, + "tokens_seen": 45940736 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980140421263792, + "loss": 4.3086, + "theoretical_loss": 5.346111706651114, + "tokens_seen": 46006272 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004980040120361084, + "loss": 4.5521, + "theoretical_loss": 5.344941234297847, + "tokens_seen": 46071808 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979939819458375, + "loss": 4.4492, + "theoretical_loss": 5.343772891165527, + "tokens_seen": 46137344 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979839518555667, + "loss": 4.4528, + "theoretical_loss": 5.34260667036464, + "tokens_seen": 46202880 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979739217652959, + "loss": 4.6275, + "theoretical_loss": 5.3414425650377, + "tokens_seen": 46268416 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979638916750251, + "loss": 4.5955, + "theoretical_loss": 5.3402805683590495, + "tokens_seen": 46333952 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979538615847543, + "loss": 4.5274, + "theoretical_loss": 5.339120673534668, + "tokens_seen": 46399488 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979438314944834, + "loss": 4.1984, + "theoretical_loss": 5.337962873801984, + "tokens_seen": 46465024 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979338014042126, + "loss": 4.4472, + "theoretical_loss": 5.336807162429681, + "tokens_seen": 46530560 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979237713139418, + "loss": 4.2688, + "theoretical_loss": 5.335653532717512, + "tokens_seen": 46596096 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497913741223671, + "loss": 4.5101, + "theoretical_loss": 5.33450197799611, + "tokens_seen": 46661632 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004979037111334002, + "loss": 4.4218, + "theoretical_loss": 5.33335249162681, + "tokens_seen": 46727168 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978936810431293, + "loss": 4.365, + "theoretical_loss": 5.332205067001454, + "tokens_seen": 46792704 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978836509528586, + "loss": 4.4012, + "theoretical_loss": 5.331059697542216, + "tokens_seen": 46858240 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978736208625877, + "loss": 4.178, + "theoretical_loss": 5.329916376701417, + "tokens_seen": 46923776 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497863590772317, + "loss": 4.404, + "theoretical_loss": 5.328775097961346, + "tokens_seen": 46989312 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978535606820461, + "loss": 4.3063, + "theoretical_loss": 5.327635854834084, + "tokens_seen": 47054848 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978435305917754, + "loss": 4.3593, + "theoretical_loss": 5.326498640861321, + "tokens_seen": 47120384 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978335005015045, + "loss": 4.458, + "theoretical_loss": 5.325363449614185, + "tokens_seen": 47185920 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978234704112337, + "loss": 4.4505, + "theoretical_loss": 5.324230274693065, + "tokens_seen": 47251456 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978134403209629, + "loss": 4.2784, + "theoretical_loss": 5.323099109727434, + "tokens_seen": 47316992 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004978034102306921, + "loss": 4.3645, + "theoretical_loss": 5.321969948375687, + "tokens_seen": 47382528 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977933801404213, + "loss": 4.3349, + "theoretical_loss": 5.320842784324954, + "tokens_seen": 47448064 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 38014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.285874366760254, + "objective/train/theoretical_loss": 5.319717611290951, + "objective/train/tokens_used": 67973600, + "theoretical_loss": 5.319717611290951, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977833500501504, + "loss": 4.2487, + "theoretical_loss": 5.319717611290951, + "tokens_seen": 47513600 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977733199598796, + "loss": 4.227, + "theoretical_loss": 5.318594423017791, + "tokens_seen": 47579136 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977632898696088, + "loss": 4.5068, + "theoretical_loss": 5.317473213277827, + "tokens_seen": 47644672 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497753259779338, + "loss": 4.4264, + "theoretical_loss": 5.316353975871486, + "tokens_seen": 47710208 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977432296890672, + "loss": 4.4848, + "theoretical_loss": 5.315236704627106, + "tokens_seen": 47775744 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977331995987965, + "loss": 4.4288, + "theoretical_loss": 5.314121393400763, + "tokens_seen": 47841280 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977231695085255, + "loss": 4.0077, + "theoretical_loss": 5.313008036076115, + "tokens_seen": 47906816 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004977131394182548, + "loss": 4.3322, + "theoretical_loss": 5.311896626564246, + "tokens_seen": 47972352 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497703109327984, + "loss": 4.1105, + "theoretical_loss": 5.3107871588034925, + "tokens_seen": 48037888 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976930792377132, + "loss": 4.3437, + "theoretical_loss": 5.309679626759296, + "tokens_seen": 48103424 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976830491474424, + "loss": 4.292, + "theoretical_loss": 5.308574024424041, + "tokens_seen": 48168960 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976730190571715, + "loss": 4.3868, + "theoretical_loss": 5.3074703458168955, + "tokens_seen": 48234496 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976629889669007, + "loss": 4.436, + "theoretical_loss": 5.3063685849836615, + "tokens_seen": 48300032 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976529588766299, + "loss": 4.368, + "theoretical_loss": 5.305268735996615, + "tokens_seen": 48365568 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976429287863591, + "loss": 4.2524, + "theoretical_loss": 5.304170792954354, + "tokens_seen": 48431104 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976328986960883, + "loss": 4.5041, + "theoretical_loss": 5.303074749981648, + "tokens_seen": 48496640 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976228686058175, + "loss": 4.412, + "theoretical_loss": 5.301980601229285, + "tokens_seen": 48562176 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976128385155466, + "loss": 4.3484, + "theoretical_loss": 5.300888340873924, + "tokens_seen": 48627712 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004976028084252758, + "loss": 4.4722, + "theoretical_loss": 5.299797963117941, + "tokens_seen": 48693248 + }, + { + "epoch": 0.01, + "learning_rate": 0.000497592778335005, + "loss": 4.1945, + "theoretical_loss": 5.298709462189283, + "tokens_seen": 48758784 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975827482447342, + "loss": 4.2003, + "theoretical_loss": 5.297622832341327, + "tokens_seen": 48824320 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975727181544635, + "loss": 4.0957, + "theoretical_loss": 5.2965380678527225, + "tokens_seen": 48889856 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975626880641925, + "loss": 4.2418, + "theoretical_loss": 5.295455163027258, + "tokens_seen": 48955392 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975526579739218, + "loss": 4.3033, + "theoretical_loss": 5.29437411219371, + "tokens_seen": 49020928 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975426278836509, + "loss": 4.2827, + "theoretical_loss": 5.293294909705702, + "tokens_seen": 49086464 + }, + { + "epoch": 0.01, + "objective/train/docs_used": 38689, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.5455546379089355, + "objective/train/theoretical_loss": 5.292217549941562, + "objective/train/tokens_used": 69612000, + "theoretical_loss": 5.292217549941562, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975325977933802, + "loss": 4.2752, + "theoretical_loss": 5.292217549941562, + "tokens_seen": 49152000 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975225677031094, + "loss": 4.1889, + "theoretical_loss": 5.291142027304185, + "tokens_seen": 49217536 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975125376128386, + "loss": 3.9788, + "theoretical_loss": 5.29006833622089, + "tokens_seen": 49283072 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004975025075225677, + "loss": 4.4666, + "theoretical_loss": 5.28899647114328, + "tokens_seen": 49348608 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974924774322969, + "loss": 4.3282, + "theoretical_loss": 5.287926426547106, + "tokens_seen": 49414144 + }, + { + "epoch": 0.01, + "learning_rate": 0.0004974824473420261, + "loss": 4.3775, + "theoretical_loss": 5.2868581969321315, + "tokens_seen": 49479680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974724172517553, + "loss": 4.3022, + "theoretical_loss": 5.285791776821995, + "tokens_seen": 49545216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974623871614845, + "loss": 4.559, + "theoretical_loss": 5.2847271607640725, + "tokens_seen": 49610752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974523570712136, + "loss": 4.5333, + "theoretical_loss": 5.283664343329346, + "tokens_seen": 49676288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974423269809428, + "loss": 4.2267, + "theoretical_loss": 5.282603319112273, + "tokens_seen": 49741824 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497432296890672, + "loss": 4.3567, + "theoretical_loss": 5.281544082730649, + "tokens_seen": 49807360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974222668004012, + "loss": 4.2938, + "theoretical_loss": 5.280486628825479, + "tokens_seen": 49872896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974122367101304, + "loss": 4.3754, + "theoretical_loss": 5.279430952060843, + "tokens_seen": 49938432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004974022066198595, + "loss": 4.3361, + "theoretical_loss": 5.278377047123777, + "tokens_seen": 50003968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973921765295888, + "loss": 4.2186, + "theoretical_loss": 5.277324908724132, + "tokens_seen": 50069504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973821464393179, + "loss": 4.2978, + "theoretical_loss": 5.276274531594453, + "tokens_seen": 50135040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973721163490472, + "loss": 4.3052, + "theoretical_loss": 5.27522591048985, + "tokens_seen": 50200576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973620862587763, + "loss": 4.352, + "theoretical_loss": 5.274179040187876, + "tokens_seen": 50266112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973520561685056, + "loss": 4.0427, + "theoretical_loss": 5.273133915488394, + "tokens_seen": 50331648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973420260782347, + "loss": 4.3078, + "theoretical_loss": 5.272090531213462, + "tokens_seen": 50397184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973319959879639, + "loss": 4.0645, + "theoretical_loss": 5.271048882207202, + "tokens_seen": 50462720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973219658976931, + "loss": 4.6409, + "theoretical_loss": 5.27000896333568, + "tokens_seen": 50528256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973119358074223, + "loss": 4.0496, + "theoretical_loss": 5.268970769486789, + "tokens_seen": 50593792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004973019057171515, + "loss": 4.4574, + "theoretical_loss": 5.267934295570118, + "tokens_seen": 50659328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972918756268806, + "loss": 4.2343, + "theoretical_loss": 5.266899536516842, + "tokens_seen": 50724864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 39885, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.198443412780762, + "objective/train/theoretical_loss": 5.265866487279597, + "objective/train/tokens_used": 71250400, + "theoretical_loss": 5.265866487279597, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972818455366098, + "loss": 4.2608, + "theoretical_loss": 5.265866487279597, + "tokens_seen": 50790400 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497271815446339, + "loss": 4.2763, + "theoretical_loss": 5.264835142832364, + "tokens_seen": 50855936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972617853560682, + "loss": 4.1906, + "theoretical_loss": 5.263805498170351, + "tokens_seen": 50921472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972517552657974, + "loss": 4.1761, + "theoretical_loss": 5.26277754830987, + "tokens_seen": 50987008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972417251755266, + "loss": 4.2526, + "theoretical_loss": 5.261751288288236, + "tokens_seen": 51052544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972316950852557, + "loss": 4.275, + "theoretical_loss": 5.260726713163637, + "tokens_seen": 51118080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972216649949849, + "loss": 4.3822, + "theoretical_loss": 5.259703818015023, + "tokens_seen": 51183616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972116349047142, + "loss": 4.235, + "theoretical_loss": 5.258682597941998, + "tokens_seen": 51249152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004972016048144433, + "loss": 4.1933, + "theoretical_loss": 5.257663048064704, + "tokens_seen": 51314688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971915747241726, + "loss": 4.0819, + "theoretical_loss": 5.256645163523704, + "tokens_seen": 51380224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971815446339017, + "loss": 4.2302, + "theoretical_loss": 5.255628939479877, + "tokens_seen": 51445760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971715145436309, + "loss": 4.1741, + "theoretical_loss": 5.254614371114306, + "tokens_seen": 51511296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971614844533601, + "loss": 4.0105, + "theoretical_loss": 5.253601453628165, + "tokens_seen": 51576832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971514543630893, + "loss": 4.1921, + "theoretical_loss": 5.2525901822426135, + "tokens_seen": 51642368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971414242728185, + "loss": 4.1033, + "theoretical_loss": 5.251580552198687, + "tokens_seen": 51707904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971313941825477, + "loss": 4.5223, + "theoretical_loss": 5.250572558757186, + "tokens_seen": 51773440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971213640922768, + "loss": 4.2685, + "theoretical_loss": 5.2495661971985745, + "tokens_seen": 51838976 + }, + { + "epoch": 0.02, + "learning_rate": 0.000497111334002006, + "loss": 4.4627, + "theoretical_loss": 5.248561462822871, + "tokens_seen": 51904512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004971013039117352, + "loss": 4.2511, + "theoretical_loss": 5.247558350949539, + "tokens_seen": 51970048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970912738214644, + "loss": 4.0554, + "theoretical_loss": 5.246556856917392, + "tokens_seen": 52035584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970812437311936, + "loss": 4.1646, + "theoretical_loss": 5.24555697608448, + "tokens_seen": 52101120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970712136409227, + "loss": 4.2295, + "theoretical_loss": 5.244558703827986, + "tokens_seen": 52166656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970611835506519, + "loss": 4.1183, + "theoretical_loss": 5.243562035544132, + "tokens_seen": 52232192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970511534603811, + "loss": 4.4799, + "theoretical_loss": 5.24256696664807, + "tokens_seen": 52297728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970411233701103, + "loss": 4.1907, + "theoretical_loss": 5.24157349257378, + "tokens_seen": 52363264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 40418, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.979517936706543, + "objective/train/theoretical_loss": 5.240581608773973, + "objective/train/tokens_used": 72888800, + "theoretical_loss": 5.240581608773973, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970310932798396, + "loss": 4.0593, + "theoretical_loss": 5.240581608773973, + "tokens_seen": 52428800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970210631895686, + "loss": 4.2029, + "theoretical_loss": 5.239591310719987, + "tokens_seen": 52494336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970110330992979, + "loss": 4.0965, + "theoretical_loss": 5.238602593901694, + "tokens_seen": 52559872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004970010030090271, + "loss": 4.0066, + "theoretical_loss": 5.23761545382739, + "tokens_seen": 52625408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969909729187563, + "loss": 4.1287, + "theoretical_loss": 5.236629886023708, + "tokens_seen": 52690944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969809428284855, + "loss": 4.1441, + "theoretical_loss": 5.235645886035517, + "tokens_seen": 52756480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969709127382147, + "loss": 4.2684, + "theoretical_loss": 5.2346634494258195, + "tokens_seen": 52822016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969608826479438, + "loss": 4.1129, + "theoretical_loss": 5.233682571775664, + "tokens_seen": 52887552 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496950852557673, + "loss": 4.44, + "theoretical_loss": 5.232703248684045, + "tokens_seen": 52953088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969408224674022, + "loss": 4.3805, + "theoretical_loss": 5.231725475767806, + "tokens_seen": 53018624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969307923771314, + "loss": 4.012, + "theoretical_loss": 5.2307492486615494, + "tokens_seen": 53084160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969207622868606, + "loss": 4.3914, + "theoretical_loss": 5.229774563017541, + "tokens_seen": 53149696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004969107321965897, + "loss": 4.2825, + "theoretical_loss": 5.2288014145056145, + "tokens_seen": 53215232 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496900702106319, + "loss": 3.9345, + "theoretical_loss": 5.227829798813083, + "tokens_seen": 53280768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968906720160481, + "loss": 4.1139, + "theoretical_loss": 5.2268597116446465, + "tokens_seen": 53346304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968806419257774, + "loss": 4.2213, + "theoretical_loss": 5.225891148722296, + "tokens_seen": 53411840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968706118355065, + "loss": 4.1851, + "theoretical_loss": 5.224924105785227, + "tokens_seen": 53477376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968605817452358, + "loss": 4.1925, + "theoretical_loss": 5.22395857858975, + "tokens_seen": 53542912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968505516549649, + "loss": 3.8839, + "theoretical_loss": 5.222994562909199, + "tokens_seen": 53608448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968405215646941, + "loss": 4.1607, + "theoretical_loss": 5.222032054533841, + "tokens_seen": 53673984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968304914744233, + "loss": 4.2475, + "theoretical_loss": 5.2210710492707895, + "tokens_seen": 53739520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968204613841525, + "loss": 3.881, + "theoretical_loss": 5.220111542943918, + "tokens_seen": 53805056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968104312938817, + "loss": 4.0171, + "theoretical_loss": 5.21915353139377, + "tokens_seen": 53870592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004968004012036108, + "loss": 4.0926, + "theoretical_loss": 5.218197010477477, + "tokens_seen": 53936128 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049679037111334, + "loss": 4.3292, + "theoretical_loss": 5.21724197606866, + "tokens_seen": 54001664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 41741, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.073785305023193, + "objective/train/theoretical_loss": 5.216288424057362, + "objective/train/tokens_used": 74527200, + "theoretical_loss": 5.216288424057362, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967803410230692, + "loss": 4.2491, + "theoretical_loss": 5.216288424057362, + "tokens_seen": 54067200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967703109327984, + "loss": 4.2446, + "theoretical_loss": 5.21533635034995, + "tokens_seen": 54132736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967602808425276, + "loss": 4.2944, + "theoretical_loss": 5.214385750869031, + "tokens_seen": 54198272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967502507522568, + "loss": 4.1296, + "theoretical_loss": 5.213436621553377, + "tokens_seen": 54263808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967402206619859, + "loss": 4.17, + "theoretical_loss": 5.212488958357831, + "tokens_seen": 54329344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967301905717151, + "loss": 4.0717, + "theoretical_loss": 5.211542757253229, + "tokens_seen": 54394880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967201604814444, + "loss": 4.4436, + "theoretical_loss": 5.210598014226319, + "tokens_seen": 54460416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967101303911735, + "loss": 4.1394, + "theoretical_loss": 5.209654725279678, + "tokens_seen": 54525952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004967001003009028, + "loss": 4.4479, + "theoretical_loss": 5.208712886431626, + "tokens_seen": 54591488 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496690070210632, + "loss": 3.9774, + "theoretical_loss": 5.207772493716154, + "tokens_seen": 54657024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966800401203611, + "loss": 4.0348, + "theoretical_loss": 5.206833543182833, + "tokens_seen": 54722560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966700100300903, + "loss": 4.1275, + "theoretical_loss": 5.205896030896748, + "tokens_seen": 54788096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966599799398195, + "loss": 4.034, + "theoretical_loss": 5.2049599529384025, + "tokens_seen": 54853632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966499498495487, + "loss": 4.2901, + "theoretical_loss": 5.204025305403653, + "tokens_seen": 54919168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966399197592779, + "loss": 4.0859, + "theoretical_loss": 5.2030920844036235, + "tokens_seen": 54984704 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496629889669007, + "loss": 4.2186, + "theoretical_loss": 5.202160286064628, + "tokens_seen": 55050240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966198595787362, + "loss": 4.1753, + "theoretical_loss": 5.2012299065281, + "tokens_seen": 55115776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004966098294884654, + "loss": 4.2037, + "theoretical_loss": 5.2003009419505055, + "tokens_seen": 55181312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965997993981946, + "loss": 4.3058, + "theoretical_loss": 5.199373388503272, + "tokens_seen": 55246848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965897693079238, + "loss": 4.3641, + "theoretical_loss": 5.198447242372716, + "tokens_seen": 55312384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965797392176529, + "loss": 4.1662, + "theoretical_loss": 5.197522499759961, + "tokens_seen": 55377920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965697091273821, + "loss": 4.1908, + "theoretical_loss": 5.196599156880863, + "tokens_seen": 55443456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965596790371113, + "loss": 3.9863, + "theoretical_loss": 5.195677209965942, + "tokens_seen": 55508992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965496489468405, + "loss": 4.5222, + "theoretical_loss": 5.194756655260305, + "tokens_seen": 55574528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965396188565698, + "loss": 3.9698, + "theoretical_loss": 5.193837489023565, + "tokens_seen": 55640064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 42409, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.305491924285889, + "objective/train/theoretical_loss": 5.192919707529784, + "objective/train/tokens_used": 76165600, + "theoretical_loss": 5.192919707529784, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965295887662988, + "loss": 4.121, + "theoretical_loss": 5.192919707529784, + "tokens_seen": 55705600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965195586760281, + "loss": 4.108, + "theoretical_loss": 5.1920033070673846, + "tokens_seen": 55771136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004965095285857573, + "loss": 3.9647, + "theoretical_loss": 5.191088283939086, + "tokens_seen": 55836672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964994984954865, + "loss": 3.9453, + "theoretical_loss": 5.190174634461828, + "tokens_seen": 55902208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964894684052157, + "loss": 3.9193, + "theoretical_loss": 5.189262354966708, + "tokens_seen": 55967744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964794383149449, + "loss": 4.0925, + "theoretical_loss": 5.188351441798897, + "tokens_seen": 56033280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496469408224674, + "loss": 4.1099, + "theoretical_loss": 5.187441891317581, + "tokens_seen": 56098816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964593781344032, + "loss": 4.0055, + "theoretical_loss": 5.186533699895888, + "tokens_seen": 56164352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964493480441324, + "loss": 4.1547, + "theoretical_loss": 5.18562686392081, + "tokens_seen": 56229888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964393179538616, + "loss": 4.1385, + "theoretical_loss": 5.184721379793145, + "tokens_seen": 56295424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964292878635908, + "loss": 4.1137, + "theoretical_loss": 5.183817243927425, + "tokens_seen": 56360960 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049641925777332, + "loss": 3.988, + "theoretical_loss": 5.182914452751846, + "tokens_seen": 56426496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004964092276830491, + "loss": 3.9087, + "theoretical_loss": 5.1820130027081985, + "tokens_seen": 56492032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963991975927783, + "loss": 4.1078, + "theoretical_loss": 5.181112890251804, + "tokens_seen": 56557568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963891675025075, + "loss": 3.9657, + "theoretical_loss": 5.180214111851447, + "tokens_seen": 56623104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963791374122367, + "loss": 3.8969, + "theoretical_loss": 5.179316663989305, + "tokens_seen": 56688640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963691073219659, + "loss": 4.0938, + "theoretical_loss": 5.178420543160889, + "tokens_seen": 56754176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963590772316951, + "loss": 4.0717, + "theoretical_loss": 5.177525745874968, + "tokens_seen": 56819712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963490471414242, + "loss": 4.0293, + "theoretical_loss": 5.176632268653516, + "tokens_seen": 56885248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963390170511535, + "loss": 4.0914, + "theoretical_loss": 5.175740108031633, + "tokens_seen": 56950784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963289869608827, + "loss": 4.3386, + "theoretical_loss": 5.174849260557494, + "tokens_seen": 57016320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004963189568706119, + "loss": 4.1438, + "theoretical_loss": 5.173959722792272, + "tokens_seen": 57081856 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496308926780341, + "loss": 4.1502, + "theoretical_loss": 5.173071491310086, + "tokens_seen": 57147392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962988966900702, + "loss": 3.9246, + "theoretical_loss": 5.172184562697928, + "tokens_seen": 57212928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962888665997994, + "loss": 4.0249, + "theoretical_loss": 5.171298933555606, + "tokens_seen": 57278464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 43695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.649152994155884, + "objective/train/theoretical_loss": 5.170414600495677, + "objective/train/tokens_used": 77804000, + "theoretical_loss": 5.170414600495677, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962788365095286, + "loss": 4.0093, + "theoretical_loss": 5.170414600495677, + "tokens_seen": 57344000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962688064192578, + "loss": 4.1786, + "theoretical_loss": 5.16953156014339, + "tokens_seen": 57409536 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496258776328987, + "loss": 4.0607, + "theoretical_loss": 5.1686498091366175, + "tokens_seen": 57475072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962487462387161, + "loss": 3.6266, + "theoretical_loss": 5.1677693441258, + "tokens_seen": 57540608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962387161484453, + "loss": 3.8942, + "theoretical_loss": 5.16689016177388, + "tokens_seen": 57606144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962286860581746, + "loss": 3.9167, + "theoretical_loss": 5.166012258756243, + "tokens_seen": 57671680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004962186559679037, + "loss": 4.2399, + "theoretical_loss": 5.16513563176066, + "tokens_seen": 57737216 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496208625877633, + "loss": 3.7377, + "theoretical_loss": 5.164260277487221, + "tokens_seen": 57802752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961985957873621, + "loss": 4.0589, + "theoretical_loss": 5.163386192648282, + "tokens_seen": 57868288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961885656970913, + "loss": 4.2436, + "theoretical_loss": 5.162513373968403, + "tokens_seen": 57933824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961785356068205, + "loss": 4.2688, + "theoretical_loss": 5.1616418181842825, + "tokens_seen": 57999360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961685055165497, + "loss": 3.9842, + "theoretical_loss": 5.160771522044712, + "tokens_seen": 58064896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961584754262789, + "loss": 4.202, + "theoretical_loss": 5.159902482310506, + "tokens_seen": 58130432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961484453360081, + "loss": 4.0252, + "theoretical_loss": 5.15903469575445, + "tokens_seen": 58195968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961384152457372, + "loss": 4.0653, + "theoretical_loss": 5.1581681591612405, + "tokens_seen": 58261504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961283851554664, + "loss": 4.1853, + "theoretical_loss": 5.15730286932743, + "tokens_seen": 58327040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961183550651956, + "loss": 4.0287, + "theoretical_loss": 5.156438823061366, + "tokens_seen": 58392576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004961083249749248, + "loss": 3.8706, + "theoretical_loss": 5.15557601718314, + "tokens_seen": 58458112 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496098294884654, + "loss": 3.4818, + "theoretical_loss": 5.154714448524525, + "tokens_seen": 58523648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960882647943831, + "loss": 4.1847, + "theoretical_loss": 5.153854113928924, + "tokens_seen": 58589184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960782347041123, + "loss": 4.0123, + "theoretical_loss": 5.152995010251313, + "tokens_seen": 58654720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960682046138415, + "loss": 3.8546, + "theoretical_loss": 5.1521371343581865, + "tokens_seen": 58720256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960581745235707, + "loss": 3.9651, + "theoretical_loss": 5.1512804831275, + "tokens_seen": 58785792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960481444333, + "loss": 3.9367, + "theoretical_loss": 5.1504250534486165, + "tokens_seen": 58851328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000496038114343029, + "loss": 4.3063, + "theoretical_loss": 5.149570842222257, + "tokens_seen": 58916864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 44395, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.063347339630127, + "objective/train/theoretical_loss": 5.1487178463604355, + "objective/train/tokens_used": 79442400, + "theoretical_loss": 5.1487178463604355, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960280842527583, + "loss": 3.7506, + "theoretical_loss": 5.1487178463604355, + "tokens_seen": 58982400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960180541624875, + "loss": 3.9872, + "theoretical_loss": 5.147866062786418, + "tokens_seen": 59047936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004960080240722167, + "loss": 3.9911, + "theoretical_loss": 5.147015488434661, + "tokens_seen": 59113472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959979939819459, + "loss": 3.8278, + "theoretical_loss": 5.14616612025076, + "tokens_seen": 59179008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959879638916751, + "loss": 3.9528, + "theoretical_loss": 5.145317955191397, + "tokens_seen": 59244544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959779338014042, + "loss": 3.7919, + "theoretical_loss": 5.144470990224288, + "tokens_seen": 59310080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959679037111334, + "loss": 3.9634, + "theoretical_loss": 5.143625222328134, + "tokens_seen": 59375616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959578736208626, + "loss": 4.096, + "theoretical_loss": 5.142780648492563, + "tokens_seen": 59441152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959478435305918, + "loss": 3.9749, + "theoretical_loss": 5.141937265718084, + "tokens_seen": 59506688 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495937813440321, + "loss": 4.0214, + "theoretical_loss": 5.141095071016032, + "tokens_seen": 59572224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959277833500501, + "loss": 3.9143, + "theoretical_loss": 5.14025406140852, + "tokens_seen": 59637760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959177532597793, + "loss": 4.0642, + "theoretical_loss": 5.139414233928385, + "tokens_seen": 59703296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004959077231695085, + "loss": 3.9695, + "theoretical_loss": 5.138575585619142, + "tokens_seen": 59768832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958976930792377, + "loss": 4.1946, + "theoretical_loss": 5.13773811353493, + "tokens_seen": 59834368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958876629889669, + "loss": 4.0784, + "theoretical_loss": 5.1369018147404635, + "tokens_seen": 59899904 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495877632898696, + "loss": 4.0826, + "theoretical_loss": 5.136066686310986, + "tokens_seen": 59965440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958676028084253, + "loss": 4.0869, + "theoretical_loss": 5.135232725332212, + "tokens_seen": 60030976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958575727181544, + "loss": 3.9758, + "theoretical_loss": 5.134399928900292, + "tokens_seen": 60096512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958475426278837, + "loss": 4.0519, + "theoretical_loss": 5.13356829412175, + "tokens_seen": 60162048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958375125376129, + "loss": 4.1545, + "theoretical_loss": 5.132737818113444, + "tokens_seen": 60227584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958274824473421, + "loss": 4.2355, + "theoretical_loss": 5.131908498002513, + "tokens_seen": 60293120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958174523570712, + "loss": 3.8189, + "theoretical_loss": 5.131080330926334, + "tokens_seen": 60358656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004958074222668004, + "loss": 4.1869, + "theoretical_loss": 5.130253314032469, + "tokens_seen": 60424192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957973921765296, + "loss": 3.728, + "theoretical_loss": 5.1294274444786225, + "tokens_seen": 60489728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957873620862588, + "loss": 4.1881, + "theoretical_loss": 5.128602719432592, + "tokens_seen": 60555264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 45571, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.731624603271484, + "objective/train/theoretical_loss": 5.12777913607222, + "objective/train/tokens_used": 81080800, + "theoretical_loss": 5.12777913607222, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495777331995988, + "loss": 4.3579, + "theoretical_loss": 5.12777913607222, + "tokens_seen": 60620800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957673019057172, + "loss": 3.8709, + "theoretical_loss": 5.126956691585351, + "tokens_seen": 60686336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957572718154463, + "loss": 3.9854, + "theoretical_loss": 5.1261353831697845, + "tokens_seen": 60751872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957472417251755, + "loss": 4.2274, + "theoretical_loss": 5.125315208033226, + "tokens_seen": 60817408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957372116349047, + "loss": 3.8993, + "theoretical_loss": 5.124496163393244, + "tokens_seen": 60882944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957271815446339, + "loss": 3.7806, + "theoretical_loss": 5.123678246477226, + "tokens_seen": 60948480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957171514543631, + "loss": 3.977, + "theoretical_loss": 5.122861454522329, + "tokens_seen": 61014016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004957071213640923, + "loss": 3.9739, + "theoretical_loss": 5.122045784775439, + "tokens_seen": 61079552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956970912738214, + "loss": 3.8256, + "theoretical_loss": 5.1212312344931235, + "tokens_seen": 61145088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956870611835507, + "loss": 4.0096, + "theoretical_loss": 5.120417800941587, + "tokens_seen": 61210624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956770310932798, + "loss": 4.0399, + "theoretical_loss": 5.119605481396629, + "tokens_seen": 61276160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956670010030091, + "loss": 3.9718, + "theoretical_loss": 5.1187942731436, + "tokens_seen": 61341696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956569709127383, + "loss": 3.9982, + "theoretical_loss": 5.117984173477356, + "tokens_seen": 61407232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956469408224674, + "loss": 4.0777, + "theoretical_loss": 5.117175179702215, + "tokens_seen": 61472768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956369107321966, + "loss": 3.9865, + "theoretical_loss": 5.116367289131917, + "tokens_seen": 61538304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956268806419258, + "loss": 4.1915, + "theoretical_loss": 5.115560499089575, + "tokens_seen": 61603840 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495616850551655, + "loss": 3.8813, + "theoretical_loss": 5.1147548069076425, + "tokens_seen": 61669376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004956068204613842, + "loss": 3.97, + "theoretical_loss": 5.113950209927859, + "tokens_seen": 61734912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955967903711133, + "loss": 3.5137, + "theoretical_loss": 5.113146705501217, + "tokens_seen": 61800448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955867602808425, + "loss": 3.7782, + "theoretical_loss": 5.112344290987915, + "tokens_seen": 61865984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955767301905717, + "loss": 4.1055, + "theoretical_loss": 5.1115429637573175, + "tokens_seen": 61931520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955667001003009, + "loss": 3.9588, + "theoretical_loss": 5.110742721187915, + "tokens_seen": 61997056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955566700100301, + "loss": 3.8558, + "theoretical_loss": 5.109943560667279, + "tokens_seen": 62062592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955466399197592, + "loss": 3.8969, + "theoretical_loss": 5.109145479592028, + "tokens_seen": 62128128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955366098294884, + "loss": 4.1078, + "theoretical_loss": 5.108348475367774, + "tokens_seen": 62193664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 46598, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.215669631958008, + "objective/train/theoretical_loss": 5.107552545409097, + "objective/train/tokens_used": 82719200, + "theoretical_loss": 5.107552545409097, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955265797392177, + "loss": 4.056, + "theoretical_loss": 5.107552545409097, + "tokens_seen": 62259200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955165496489468, + "loss": 3.9227, + "theoretical_loss": 5.106757687139494, + "tokens_seen": 62324736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004955065195586761, + "loss": 4.2246, + "theoretical_loss": 5.1059638979913435, + "tokens_seen": 62390272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954964894684052, + "loss": 4.0438, + "theoretical_loss": 5.105171175405864, + "tokens_seen": 62455808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954864593781344, + "loss": 4.0275, + "theoretical_loss": 5.104379516833075, + "tokens_seen": 62521344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954764292878636, + "loss": 4.005, + "theoretical_loss": 5.103588919731757, + "tokens_seen": 62586880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954663991975928, + "loss": 3.937, + "theoretical_loss": 5.1027993815694135, + "tokens_seen": 62652416 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495456369107322, + "loss": 4.1536, + "theoretical_loss": 5.10201089982223, + "tokens_seen": 62717952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954463390170512, + "loss": 3.971, + "theoretical_loss": 5.101223471975038, + "tokens_seen": 62783488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954363089267803, + "loss": 3.9062, + "theoretical_loss": 5.100437095521272, + "tokens_seen": 62849024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954262788365095, + "loss": 3.8866, + "theoretical_loss": 5.099651767962936, + "tokens_seen": 62914560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954162487462387, + "loss": 3.8535, + "theoretical_loss": 5.098867486810562, + "tokens_seen": 62980096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004954062186559679, + "loss": 3.7859, + "theoretical_loss": 5.098084249583175, + "tokens_seen": 63045632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953961885656971, + "loss": 3.9093, + "theoretical_loss": 5.097302053808251, + "tokens_seen": 63111168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953861584754263, + "loss": 3.5672, + "theoretical_loss": 5.0965208970216835, + "tokens_seen": 63176704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953761283851555, + "loss": 3.8846, + "theoretical_loss": 5.095740776767743, + "tokens_seen": 63242240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953660982948846, + "loss": 3.9469, + "theoretical_loss": 5.0949616905990425, + "tokens_seen": 63307776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953560682046139, + "loss": 4.2765, + "theoretical_loss": 5.094183636076501, + "tokens_seen": 63373312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953460381143431, + "loss": 4.015, + "theoretical_loss": 5.093406610769302, + "tokens_seen": 63438848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953360080240723, + "loss": 4.2246, + "theoretical_loss": 5.092630612254862, + "tokens_seen": 63504384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953259779338014, + "loss": 3.9173, + "theoretical_loss": 5.091855638118792, + "tokens_seen": 63569920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953159478435306, + "loss": 4.0249, + "theoretical_loss": 5.091081685954863, + "tokens_seen": 63635456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004953059177532598, + "loss": 3.7978, + "theoretical_loss": 5.090308753364967, + "tokens_seen": 63700992 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495295887662989, + "loss": 3.8605, + "theoretical_loss": 5.089536837959084, + "tokens_seen": 63766528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952858575727182, + "loss": 3.9657, + "theoretical_loss": 5.0887659373552445, + "tokens_seen": 63832064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 47189, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3733479976654053, + "objective/train/theoretical_loss": 5.087996049179497, + "objective/train/tokens_used": 84357600, + "theoretical_loss": 5.087996049179497, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952758274824474, + "loss": 3.6824, + "theoretical_loss": 5.087996049179497, + "tokens_seen": 63897600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952657973921765, + "loss": 3.7677, + "theoretical_loss": 5.087227171065869, + "tokens_seen": 63963136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952557673019057, + "loss": 3.9197, + "theoretical_loss": 5.086459300656337, + "tokens_seen": 64028672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952457372116349, + "loss": 3.927, + "theoretical_loss": 5.085692435600783, + "tokens_seen": 64094208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952357071213641, + "loss": 3.8821, + "theoretical_loss": 5.084926573556972, + "tokens_seen": 64159744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952256770310933, + "loss": 3.7931, + "theoretical_loss": 5.084161712190507, + "tokens_seen": 64225280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952156469408225, + "loss": 3.7647, + "theoretical_loss": 5.083397849174801, + "tokens_seen": 64290816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004952056168505516, + "loss": 3.9431, + "theoretical_loss": 5.082634982191042, + "tokens_seen": 64356352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951955867602809, + "loss": 3.9173, + "theoretical_loss": 5.081873108928154, + "tokens_seen": 64421888 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049518555667001, + "loss": 4.0204, + "theoretical_loss": 5.081112227082771, + "tokens_seen": 64487424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951755265797393, + "loss": 3.816, + "theoretical_loss": 5.0803523343591985, + "tokens_seen": 64552960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951654964894685, + "loss": 3.8761, + "theoretical_loss": 5.079593428469382, + "tokens_seen": 64618496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951554663991976, + "loss": 4.0756, + "theoretical_loss": 5.078835507132873, + "tokens_seen": 64684032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951454363089268, + "loss": 3.8136, + "theoretical_loss": 5.078078568076798, + "tokens_seen": 64749568 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495135406218656, + "loss": 4.0581, + "theoretical_loss": 5.077322609035821, + "tokens_seen": 64815104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951253761283852, + "loss": 3.8215, + "theoretical_loss": 5.076567627752118, + "tokens_seen": 64880640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951153460381144, + "loss": 3.8355, + "theoretical_loss": 5.075813621975337, + "tokens_seen": 64946176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004951053159478435, + "loss": 3.9719, + "theoretical_loss": 5.075060589462572, + "tokens_seen": 65011712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950952858575727, + "loss": 3.7383, + "theoretical_loss": 5.074308527978326, + "tokens_seen": 65077248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950852557673019, + "loss": 3.8755, + "theoretical_loss": 5.073557435294486, + "tokens_seen": 65142784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950752256770311, + "loss": 3.6252, + "theoretical_loss": 5.07280730919028, + "tokens_seen": 65208320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950651955867603, + "loss": 3.7529, + "theoretical_loss": 5.072058147452257, + "tokens_seen": 65273856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950551654964894, + "loss": 3.679, + "theoretical_loss": 5.071309947874248, + "tokens_seen": 65339392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950451354062186, + "loss": 3.5894, + "theoretical_loss": 5.070562708257339, + "tokens_seen": 65404928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950351053159479, + "loss": 3.9951, + "theoretical_loss": 5.069816426409837, + "tokens_seen": 65470464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 48433, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.8971009254455566, + "objective/train/theoretical_loss": 5.069071100147241, + "objective/train/tokens_used": 85996000, + "theoretical_loss": 5.069071100147241, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.000495025075225677, + "loss": 3.9133, + "theoretical_loss": 5.069071100147241, + "tokens_seen": 65536000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950150451354063, + "loss": 4.2034, + "theoretical_loss": 5.068326727292211, + "tokens_seen": 65601536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004950050150451354, + "loss": 3.7703, + "theoretical_loss": 5.067583305674537, + "tokens_seen": 65667072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949949849548646, + "loss": 3.8678, + "theoretical_loss": 5.066840833131106, + "tokens_seen": 65732608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949849548645938, + "loss": 3.8256, + "theoretical_loss": 5.066099307505878, + "tokens_seen": 65798144 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494974924774323, + "loss": 3.6242, + "theoretical_loss": 5.06535872664985, + "tokens_seen": 65863680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949648946840522, + "loss": 3.9033, + "theoretical_loss": 5.064619088421029, + "tokens_seen": 65929216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949548645937814, + "loss": 3.5217, + "theoretical_loss": 5.0638803906844, + "tokens_seen": 65994752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949448345035105, + "loss": 3.8469, + "theoretical_loss": 5.0631426313119, + "tokens_seen": 66060288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949348044132397, + "loss": 3.7472, + "theoretical_loss": 5.062405808182383, + "tokens_seen": 66125824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949247743229689, + "loss": 3.4883, + "theoretical_loss": 5.061669919181599, + "tokens_seen": 66191360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949147442326981, + "loss": 3.8441, + "theoretical_loss": 5.060934962202155, + "tokens_seen": 66256896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004949047141424273, + "loss": 3.9786, + "theoretical_loss": 5.060200935143492, + "tokens_seen": 66322432 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948946840521565, + "loss": 4.0608, + "theoretical_loss": 5.059467835911857, + "tokens_seen": 66387968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948846539618856, + "loss": 3.7521, + "theoretical_loss": 5.058735662420268, + "tokens_seen": 66453504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948746238716148, + "loss": 3.6167, + "theoretical_loss": 5.0580044125884935, + "tokens_seen": 66519040 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494864593781344, + "loss": 3.76, + "theoretical_loss": 5.057274084343018, + "tokens_seen": 66584576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948545636910733, + "loss": 3.8083, + "theoretical_loss": 5.056544675617017, + "tokens_seen": 66650112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948445336008024, + "loss": 3.7601, + "theoretical_loss": 5.055816184350326, + "tokens_seen": 66715648 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948345035105316, + "loss": 3.8069, + "theoretical_loss": 5.055088608489417, + "tokens_seen": 66781184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948244734202607, + "loss": 3.3711, + "theoretical_loss": 5.054361945987363, + "tokens_seen": 66846720 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049481444332999, + "loss": 3.3429, + "theoretical_loss": 5.05363619480382, + "tokens_seen": 66912256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004948044132397192, + "loss": 4.0226, + "theoretical_loss": 5.052911352904992, + "tokens_seen": 66977792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947943831494484, + "loss": 3.7436, + "theoretical_loss": 5.052187418263609, + "tokens_seen": 67043328 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947843530591776, + "loss": 3.8333, + "theoretical_loss": 5.051464388858893, + "tokens_seen": 67108864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 48877, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7897393703460693, + "objective/train/theoretical_loss": 5.050742262676536, + "objective/train/tokens_used": 87634400, + "theoretical_loss": 5.050742262676536, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947743229689067, + "loss": 3.7583, + "theoretical_loss": 5.050742262676536, + "tokens_seen": 67174400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947642928786359, + "loss": 4.0895, + "theoretical_loss": 5.050021037708673, + "tokens_seen": 67239936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947542627883651, + "loss": 3.642, + "theoretical_loss": 5.049300711953854, + "tokens_seen": 67305472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947442326980943, + "loss": 3.7434, + "theoretical_loss": 5.048581283417015, + "tokens_seen": 67371008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947342026078235, + "loss": 3.7182, + "theoretical_loss": 5.047862750109456, + "tokens_seen": 67436544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947241725175527, + "loss": 3.9326, + "theoretical_loss": 5.047145110048811, + "tokens_seen": 67502080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947141424272818, + "loss": 3.638, + "theoretical_loss": 5.046428361259022, + "tokens_seen": 67567616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004947041123370111, + "loss": 3.7471, + "theoretical_loss": 5.045712501770317, + "tokens_seen": 67633152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946940822467402, + "loss": 3.699, + "theoretical_loss": 5.0449975296191765, + "tokens_seen": 67698688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946840521564695, + "loss": 3.8026, + "theoretical_loss": 5.0442834428483145, + "tokens_seen": 67764224 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946740220661987, + "loss": 3.7185, + "theoretical_loss": 5.043570239506648, + "tokens_seen": 67829760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946639919759278, + "loss": 3.8962, + "theoretical_loss": 5.042857917649275, + "tokens_seen": 67895296 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494653961885657, + "loss": 3.7692, + "theoretical_loss": 5.042146475337448, + "tokens_seen": 67960832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946439317953862, + "loss": 3.5141, + "theoretical_loss": 5.041435910638544, + "tokens_seen": 68026368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946339017051154, + "loss": 3.8396, + "theoretical_loss": 5.040726221626048, + "tokens_seen": 68091904 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946238716148446, + "loss": 3.8633, + "theoretical_loss": 5.040017406379521, + "tokens_seen": 68157440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946138415245737, + "loss": 3.6866, + "theoretical_loss": 5.0393094629845745, + "tokens_seen": 68222976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004946038114343029, + "loss": 3.5706, + "theoretical_loss": 5.038602389532853, + "tokens_seen": 68288512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945937813440321, + "loss": 3.4773, + "theoretical_loss": 5.037896184122, + "tokens_seen": 68354048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945837512537613, + "loss": 3.7606, + "theoretical_loss": 5.037190844855641, + "tokens_seen": 68419584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945737211634905, + "loss": 3.8062, + "theoretical_loss": 5.036486369843355, + "tokens_seen": 68485120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945636910732196, + "loss": 3.6164, + "theoretical_loss": 5.03578275720065, + "tokens_seen": 68550656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945536609829488, + "loss": 3.7046, + "theoretical_loss": 5.03508000504894, + "tokens_seen": 68616192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945436308926781, + "loss": 3.7478, + "theoretical_loss": 5.034378111515521, + "tokens_seen": 68681728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945336008024072, + "loss": 3.663, + "theoretical_loss": 5.0336770747335455, + "tokens_seen": 68747264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 49922, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9727299213409424, + "objective/train/theoretical_loss": 5.032976892842002, + "objective/train/tokens_used": 89272800, + "theoretical_loss": 5.032976892842002, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945235707121365, + "loss": 3.6731, + "theoretical_loss": 5.032976892842002, + "tokens_seen": 68812800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945135406218656, + "loss": 3.8125, + "theoretical_loss": 5.032277563985687, + "tokens_seen": 68878336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004945035105315948, + "loss": 3.8008, + "theoretical_loss": 5.031579086315183, + "tokens_seen": 68943872 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494493480441324, + "loss": 3.9551, + "theoretical_loss": 5.030881457986837, + "tokens_seen": 69009408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944834503510532, + "loss": 3.843, + "theoretical_loss": 5.030184677162734, + "tokens_seen": 69074944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944734202607824, + "loss": 3.5448, + "theoretical_loss": 5.029488742010676, + "tokens_seen": 69140480 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944633901705116, + "loss": 3.8376, + "theoretical_loss": 5.028793650704154, + "tokens_seen": 69206016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944533600802407, + "loss": 3.9125, + "theoretical_loss": 5.028099401422336, + "tokens_seen": 69271552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944433299899699, + "loss": 3.6831, + "theoretical_loss": 5.0274059923500305, + "tokens_seen": 69337088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944332998996991, + "loss": 3.7007, + "theoretical_loss": 5.026713421677673, + "tokens_seen": 69402624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944232698094283, + "loss": 3.6241, + "theoretical_loss": 5.0260216876013, + "tokens_seen": 69468160 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944132397191575, + "loss": 3.7006, + "theoretical_loss": 5.025330788322526, + "tokens_seen": 69533696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004944032096288867, + "loss": 3.9117, + "theoretical_loss": 5.024640722048522, + "tokens_seen": 69599232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943931795386158, + "loss": 3.4086, + "theoretical_loss": 5.023951486991994, + "tokens_seen": 69664768 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494383149448345, + "loss": 3.9368, + "theoretical_loss": 5.023263081371157, + "tokens_seen": 69730304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943731193580742, + "loss": 3.8784, + "theoretical_loss": 5.022575503409717, + "tokens_seen": 69795840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943630892678035, + "loss": 3.941, + "theoretical_loss": 5.021888751336848, + "tokens_seen": 69861376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943530591775326, + "loss": 3.8625, + "theoretical_loss": 5.02120282338717, + "tokens_seen": 69926912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943430290872618, + "loss": 3.6762, + "theoretical_loss": 5.020517717800723, + "tokens_seen": 69992448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943329989969909, + "loss": 3.4889, + "theoretical_loss": 5.019833432822953, + "tokens_seen": 70057984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943229689067202, + "loss": 3.5122, + "theoretical_loss": 5.019149966704683, + "tokens_seen": 70123520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943129388164494, + "loss": 3.6293, + "theoretical_loss": 5.018467317702097, + "tokens_seen": 70189056 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004943029087261786, + "loss": 3.5085, + "theoretical_loss": 5.017785484076715, + "tokens_seen": 70254592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942928786359078, + "loss": 3.6348, + "theoretical_loss": 5.017104464095375, + "tokens_seen": 70320128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942828485456369, + "loss": 3.6907, + "theoretical_loss": 5.0164242560302075, + "tokens_seen": 70385664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 51260, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.9014105796813965, + "objective/train/theoretical_loss": 5.015744858158618, + "objective/train/tokens_used": 90911200, + "theoretical_loss": 5.015744858158618, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942728184553661, + "loss": 3.8765, + "theoretical_loss": 5.015744858158618, + "tokens_seen": 70451200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942627883650953, + "loss": 3.8516, + "theoretical_loss": 5.015066268763264, + "tokens_seen": 70516736 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942527582748245, + "loss": 3.6519, + "theoretical_loss": 5.014388486132036, + "tokens_seen": 70582272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942427281845537, + "loss": 3.9132, + "theoretical_loss": 5.0137115085580355, + "tokens_seen": 70647808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942326980942828, + "loss": 3.8834, + "theoretical_loss": 5.013035334339554, + "tokens_seen": 70713344 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494222668004012, + "loss": 3.8794, + "theoretical_loss": 5.012359961780053, + "tokens_seen": 70778880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942126379137412, + "loss": 3.8483, + "theoretical_loss": 5.011685389188142, + "tokens_seen": 70844416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004942026078234704, + "loss": 3.4989, + "theoretical_loss": 5.011011614877562, + "tokens_seen": 70909952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941925777331996, + "loss": 3.7672, + "theoretical_loss": 5.010338637167162, + "tokens_seen": 70975488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941825476429289, + "loss": 3.6359, + "theoretical_loss": 5.009666454380877, + "tokens_seen": 71041024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941725175526579, + "loss": 3.8619, + "theoretical_loss": 5.008995064847716, + "tokens_seen": 71106560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941624874623872, + "loss": 3.8008, + "theoretical_loss": 5.0083244669017315, + "tokens_seen": 71172096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941524573721163, + "loss": 3.9147, + "theoretical_loss": 5.0076546588820054, + "tokens_seen": 71237632 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941424272818456, + "loss": 3.3797, + "theoretical_loss": 5.00698563913263, + "tokens_seen": 71303168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941323971915748, + "loss": 3.617, + "theoretical_loss": 5.006317406002687, + "tokens_seen": 71368704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941223671013039, + "loss": 3.5556, + "theoretical_loss": 5.0056499578462255, + "tokens_seen": 71434240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941123370110331, + "loss": 3.4481, + "theoretical_loss": 5.0049832930222475, + "tokens_seen": 71499776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004941023069207623, + "loss": 3.826, + "theoretical_loss": 5.004317409894682, + "tokens_seen": 71565312 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940922768304915, + "loss": 3.6718, + "theoretical_loss": 5.003652306832372, + "tokens_seen": 71630848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940822467402207, + "loss": 3.8578, + "theoretical_loss": 5.002987982209052, + "tokens_seen": 71696384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940722166499498, + "loss": 3.758, + "theoretical_loss": 5.00232443440333, + "tokens_seen": 71761920 + }, + { + "epoch": 0.02, + "learning_rate": 0.000494062186559679, + "loss": 3.4768, + "theoretical_loss": 5.001661661798666, + "tokens_seen": 71827456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940521564694082, + "loss": 3.6442, + "theoretical_loss": 5.000999662783359, + "tokens_seen": 71892992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940421263791374, + "loss": 3.7435, + "theoretical_loss": 5.0003384357505185, + "tokens_seen": 71958528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940320962888666, + "loss": 3.8745, + "theoretical_loss": 4.999677979098056, + "tokens_seen": 72024064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 51878, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4065515995025635, + "objective/train/theoretical_loss": 4.999018291228659, + "objective/train/tokens_used": 92549600, + "theoretical_loss": 4.999018291228659, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940220661985958, + "loss": 3.5808, + "theoretical_loss": 4.999018291228659, + "tokens_seen": 72089600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940120361083249, + "loss": 3.8043, + "theoretical_loss": 4.998359370549778, + "tokens_seen": 72155136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004940020060180542, + "loss": 3.649, + "theoretical_loss": 4.997701215473601, + "tokens_seen": 72220672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939919759277834, + "loss": 3.6107, + "theoretical_loss": 4.997043824417044, + "tokens_seen": 72286208 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939819458375126, + "loss": 3.6793, + "theoretical_loss": 4.996387195801723, + "tokens_seen": 72351744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939719157472418, + "loss": 3.6529, + "theoretical_loss": 4.995731328053944, + "tokens_seen": 72417280 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493961885656971, + "loss": 3.7394, + "theoretical_loss": 4.995076219604682, + "tokens_seen": 72482816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939518555667001, + "loss": 3.6256, + "theoretical_loss": 4.994421868889559, + "tokens_seen": 72548352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939418254764293, + "loss": 3.6209, + "theoretical_loss": 4.9937682743488345, + "tokens_seen": 72613888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939317953861585, + "loss": 3.5004, + "theoretical_loss": 4.993115434427376, + "tokens_seen": 72679424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939217652958877, + "loss": 3.5103, + "theoretical_loss": 4.992463347574654, + "tokens_seen": 72744960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004939117352056169, + "loss": 3.7514, + "theoretical_loss": 4.991812012244717, + "tokens_seen": 72810496 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493901705115346, + "loss": 3.5319, + "theoretical_loss": 4.99116142689617, + "tokens_seen": 72876032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938916750250752, + "loss": 3.8628, + "theoretical_loss": 4.990511589992169, + "tokens_seen": 72941568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938816449348044, + "loss": 4.0163, + "theoretical_loss": 4.989862500000392, + "tokens_seen": 73007104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938716148445337, + "loss": 3.7264, + "theoretical_loss": 4.989214155393025, + "tokens_seen": 73072640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938615847542628, + "loss": 3.9026, + "theoretical_loss": 4.988566554646752, + "tokens_seen": 73138176 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493851554663992, + "loss": 3.4925, + "theoretical_loss": 4.9879196962427255, + "tokens_seen": 73203712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938415245737211, + "loss": 3.5623, + "theoretical_loss": 4.9872735786665565, + "tokens_seen": 73269248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938314944834504, + "loss": 3.415, + "theoretical_loss": 4.986628200408299, + "tokens_seen": 73334784 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938214643931796, + "loss": 3.5253, + "theoretical_loss": 4.985983559962429, + "tokens_seen": 73400320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004938114343029088, + "loss": 3.5016, + "theoretical_loss": 4.985339655827827, + "tokens_seen": 73465856 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493801404212638, + "loss": 3.8156, + "theoretical_loss": 4.984696486507767, + "tokens_seen": 73531392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937913741223671, + "loss": 3.7367, + "theoretical_loss": 4.984054050509893, + "tokens_seen": 73596928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937813440320963, + "loss": 3.6885, + "theoretical_loss": 4.983412346346206, + "tokens_seen": 73662464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 53017, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.889392137527466, + "objective/train/theoretical_loss": 4.982771372533048, + "objective/train/tokens_used": 94188000, + "theoretical_loss": 4.982771372533048, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937713139418255, + "loss": 3.6721, + "theoretical_loss": 4.982771372533048, + "tokens_seen": 73728000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937612838515547, + "loss": 3.9426, + "theoretical_loss": 4.982131127591083, + "tokens_seen": 73793536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937512537612839, + "loss": 3.6135, + "theoretical_loss": 4.981491610045282, + "tokens_seen": 73859072 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493741223671013, + "loss": 3.6257, + "theoretical_loss": 4.980852818424905, + "tokens_seen": 73924608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937311935807422, + "loss": 3.6675, + "theoretical_loss": 4.980214751263491, + "tokens_seen": 73990144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937211634904714, + "loss": 3.6453, + "theoretical_loss": 4.979577407098831, + "tokens_seen": 74055680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937111334002006, + "loss": 3.7323, + "theoretical_loss": 4.978940784472962, + "tokens_seen": 74121216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004937011033099298, + "loss": 3.4667, + "theoretical_loss": 4.978304881932145, + "tokens_seen": 74186752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936910732196591, + "loss": 3.9063, + "theoretical_loss": 4.97766969802685, + "tokens_seen": 74252288 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936810431293881, + "loss": 3.3322, + "theoretical_loss": 4.977035231311745, + "tokens_seen": 74317824 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936710130391174, + "loss": 3.6439, + "theoretical_loss": 4.976401480345672, + "tokens_seen": 74383360 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936609829488465, + "loss": 3.5552, + "theoretical_loss": 4.975768443691636, + "tokens_seen": 74448896 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936509528585758, + "loss": 3.5095, + "theoretical_loss": 4.97513611991679, + "tokens_seen": 74514432 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493640922768305, + "loss": 3.7738, + "theoretical_loss": 4.974504507592419, + "tokens_seen": 74579968 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936308926780341, + "loss": 3.8278, + "theoretical_loss": 4.973873605293922, + "tokens_seen": 74645504 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936208625877633, + "loss": 3.7248, + "theoretical_loss": 4.973243411600797, + "tokens_seen": 74711040 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936108324974925, + "loss": 3.8762, + "theoretical_loss": 4.9726139250966295, + "tokens_seen": 74776576 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004936008024072217, + "loss": 3.6798, + "theoretical_loss": 4.971985144369073, + "tokens_seen": 74842112 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935907723169509, + "loss": 3.6282, + "theoretical_loss": 4.9713570680098345, + "tokens_seen": 74907648 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049358074222668, + "loss": 3.7215, + "theoretical_loss": 4.970729694614662, + "tokens_seen": 74973184 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935707121364092, + "loss": 3.5638, + "theoretical_loss": 4.970103022783325, + "tokens_seen": 75038720 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935606820461384, + "loss": 3.5533, + "theoretical_loss": 4.969477051119606, + "tokens_seen": 75104256 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935506519558676, + "loss": 3.496, + "theoretical_loss": 4.968851778231276, + "tokens_seen": 75169792 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935406218655968, + "loss": 3.6459, + "theoretical_loss": 4.9682272027300876, + "tokens_seen": 75235328 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493530591775326, + "loss": 3.5601, + "theoretical_loss": 4.96760332323176, + "tokens_seen": 75300864 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 53591, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.8066093921661377, + "objective/train/theoretical_loss": 4.966980138355957, + "objective/train/tokens_used": 95826400, + "theoretical_loss": 4.966980138355957, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935205616850551, + "loss": 3.7377, + "theoretical_loss": 4.966980138355957, + "tokens_seen": 75366400 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935105315947844, + "loss": 3.6051, + "theoretical_loss": 4.966357646726282, + "tokens_seen": 75431936 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004935005015045135, + "loss": 3.5847, + "theoretical_loss": 4.965735846970255, + "tokens_seen": 75497472 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934904714142428, + "loss": 3.7027, + "theoretical_loss": 4.965114737719302, + "tokens_seen": 75563008 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934804413239719, + "loss": 3.648, + "theoretical_loss": 4.964494317608743, + "tokens_seen": 75628544 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934704112337011, + "loss": 3.4989, + "theoretical_loss": 4.963874585277772, + "tokens_seen": 75694080 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934603811434303, + "loss": 3.8889, + "theoretical_loss": 4.963255539369445, + "tokens_seen": 75759616 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934503510531595, + "loss": 3.5961, + "theoretical_loss": 4.9626371785306675, + "tokens_seen": 75825152 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934403209628887, + "loss": 3.4261, + "theoretical_loss": 4.962019501412178, + "tokens_seen": 75890688 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934302908726179, + "loss": 3.7473, + "theoretical_loss": 4.961402506668535, + "tokens_seen": 75956224 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493420260782347, + "loss": 3.4055, + "theoretical_loss": 4.960786192958103, + "tokens_seen": 76021760 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934102306920762, + "loss": 3.6034, + "theoretical_loss": 4.960170558943037, + "tokens_seen": 76087296 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004934002006018054, + "loss": 3.4187, + "theoretical_loss": 4.959555603289269, + "tokens_seen": 76152832 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933901705115346, + "loss": 3.6193, + "theoretical_loss": 4.9589413246664975, + "tokens_seen": 76218368 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933801404212638, + "loss": 3.794, + "theoretical_loss": 4.958327721748168, + "tokens_seen": 76283904 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493370110330993, + "loss": 3.3982, + "theoretical_loss": 4.957714793211462, + "tokens_seen": 76349440 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933600802407221, + "loss": 3.3334, + "theoretical_loss": 4.957102537737284, + "tokens_seen": 76414976 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933500501504513, + "loss": 3.6421, + "theoretical_loss": 4.956490954010249, + "tokens_seen": 76480512 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933400200601805, + "loss": 3.5119, + "theoretical_loss": 4.9558800407186645, + "tokens_seen": 76546048 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933299899699098, + "loss": 3.4933, + "theoretical_loss": 4.95526979655452, + "tokens_seen": 76611584 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933199598796389, + "loss": 3.7028, + "theoretical_loss": 4.954660220213473, + "tokens_seen": 76677120 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004933099297893682, + "loss": 3.7165, + "theoretical_loss": 4.954051310394836, + "tokens_seen": 76742656 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932998996990972, + "loss": 3.5507, + "theoretical_loss": 4.953443065801563, + "tokens_seen": 76808192 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932898696088265, + "loss": 3.4901, + "theoretical_loss": 4.952835485140236, + "tokens_seen": 76873728 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932798395185557, + "loss": 3.2798, + "theoretical_loss": 4.952228567121052, + "tokens_seen": 76939264 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 54447, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6758010387420654, + "objective/train/theoretical_loss": 4.951622310457807, + "objective/train/tokens_used": 97464800, + "theoretical_loss": 4.951622310457807, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932698094282849, + "loss": 3.4817, + "theoretical_loss": 4.951622310457807, + "tokens_seen": 77004800 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932597793380141, + "loss": 3.5561, + "theoretical_loss": 4.95101671386789, + "tokens_seen": 77070336 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932497492477432, + "loss": 3.6278, + "theoretical_loss": 4.950411776072261, + "tokens_seen": 77135872 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932397191574724, + "loss": 3.7609, + "theoretical_loss": 4.949807495795445, + "tokens_seen": 77201408 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932296890672016, + "loss": 3.6186, + "theoretical_loss": 4.949203871765516, + "tokens_seen": 77266944 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004932196589769308, + "loss": 3.6535, + "theoretical_loss": 4.948600902714083, + "tokens_seen": 77332480 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049320962888666, + "loss": 3.4356, + "theoretical_loss": 4.947998587376282, + "tokens_seen": 77398016 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931995987963893, + "loss": 3.7607, + "theoretical_loss": 4.947396924490757, + "tokens_seen": 77463552 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931895687061183, + "loss": 3.9334, + "theoretical_loss": 4.946795912799653, + "tokens_seen": 77529088 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931795386158476, + "loss": 3.4391, + "theoretical_loss": 4.946195551048597, + "tokens_seen": 77594624 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931695085255767, + "loss": 3.4097, + "theoretical_loss": 4.945595837986693, + "tokens_seen": 77660160 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493159478435306, + "loss": 3.4903, + "theoretical_loss": 4.944996772366503, + "tokens_seen": 77725696 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931494483450352, + "loss": 3.5634, + "theoretical_loss": 4.944398352944038, + "tokens_seen": 77791232 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931394182547643, + "loss": 3.5448, + "theoretical_loss": 4.943800578478745, + "tokens_seen": 77856768 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931293881644935, + "loss": 3.4788, + "theoretical_loss": 4.9432034477334925, + "tokens_seen": 77922304 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931193580742227, + "loss": 3.4963, + "theoretical_loss": 4.942606959474562, + "tokens_seen": 77987840 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004931093279839519, + "loss": 3.4853, + "theoretical_loss": 4.942011112471633, + "tokens_seen": 78053376 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930992978936811, + "loss": 3.3058, + "theoretical_loss": 4.94141590549777, + "tokens_seen": 78118912 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930892678034102, + "loss": 3.7623, + "theoretical_loss": 4.940821337329414, + "tokens_seen": 78184448 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930792377131394, + "loss": 3.324, + "theoretical_loss": 4.940227406746368, + "tokens_seen": 78249984 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930692076228686, + "loss": 3.595, + "theoretical_loss": 4.939634112531784, + "tokens_seen": 78315520 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930591775325978, + "loss": 3.6571, + "theoretical_loss": 4.939041453472152, + "tokens_seen": 78381056 + }, + { + "epoch": 0.02, + "learning_rate": 0.000493049147442327, + "loss": 3.546, + "theoretical_loss": 4.9384494283572895, + "tokens_seen": 78446592 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930391173520562, + "loss": 3.7395, + "theoretical_loss": 4.937858035980327, + "tokens_seen": 78512128 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930290872617853, + "loss": 3.5258, + "theoretical_loss": 4.9372672751377, + "tokens_seen": 78577664 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 54900, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9137701988220215, + "objective/train/theoretical_loss": 4.936677144629131, + "objective/train/tokens_used": 99103200, + "theoretical_loss": 4.936677144629131, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930190571715146, + "loss": 3.2102, + "theoretical_loss": 4.936677144629131, + "tokens_seen": 78643200 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004930090270812437, + "loss": 3.7118, + "theoretical_loss": 4.936087643257625, + "tokens_seen": 78708736 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492998996990973, + "loss": 3.5876, + "theoretical_loss": 4.935498769829453, + "tokens_seen": 78774272 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929889669007021, + "loss": 3.7919, + "theoretical_loss": 4.934910523154139, + "tokens_seen": 78839808 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929789368104313, + "loss": 3.8801, + "theoretical_loss": 4.934322902044457, + "tokens_seen": 78905344 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929689067201605, + "loss": 3.5096, + "theoretical_loss": 4.933735905316408, + "tokens_seen": 78970880 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929588766298897, + "loss": 3.4217, + "theoretical_loss": 4.933149531789217, + "tokens_seen": 79036416 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929488465396189, + "loss": 3.6953, + "theoretical_loss": 4.932563780285319, + "tokens_seen": 79101952 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929388164493481, + "loss": 3.6487, + "theoretical_loss": 4.931978649630346, + "tokens_seen": 79167488 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929287863590773, + "loss": 3.6641, + "theoretical_loss": 4.931394138653117, + "tokens_seen": 79233024 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929187562688064, + "loss": 3.8094, + "theoretical_loss": 4.930810246185626, + "tokens_seen": 79298560 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004929087261785356, + "loss": 3.5257, + "theoretical_loss": 4.930226971063035, + "tokens_seen": 79364096 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928986960882648, + "loss": 3.4985, + "theoretical_loss": 4.929644312123655, + "tokens_seen": 79429632 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492888665997994, + "loss": 3.3258, + "theoretical_loss": 4.929062268208941, + "tokens_seen": 79495168 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928786359077232, + "loss": 3.2647, + "theoretical_loss": 4.92848083816348, + "tokens_seen": 79560704 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928686058174523, + "loss": 3.8228, + "theoretical_loss": 4.927900020834979, + "tokens_seen": 79626240 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928585757271815, + "loss": 3.3943, + "theoretical_loss": 4.927319815074252, + "tokens_seen": 79691776 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928485456369107, + "loss": 3.4187, + "theoretical_loss": 4.926740219735214, + "tokens_seen": 79757312 + }, + { + "epoch": 0.02, + "learning_rate": 0.00049283851554664, + "loss": 3.6324, + "theoretical_loss": 4.926161233674863, + "tokens_seen": 79822848 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928284854563691, + "loss": 3.627, + "theoretical_loss": 4.925582855753279, + "tokens_seen": 79888384 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928184553660984, + "loss": 3.4882, + "theoretical_loss": 4.925005084833603, + "tokens_seen": 79953920 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004928084252758274, + "loss": 3.8262, + "theoretical_loss": 4.924427919782033, + "tokens_seen": 80019456 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927983951855567, + "loss": 3.5192, + "theoretical_loss": 4.923851359467811, + "tokens_seen": 80084992 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927883650952859, + "loss": 3.714, + "theoretical_loss": 4.923275402763213, + "tokens_seen": 80150528 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927783350050151, + "loss": 3.4187, + "theoretical_loss": 4.922700048543536, + "tokens_seen": 80216064 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 56090, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1410176753997803, + "objective/train/theoretical_loss": 4.922125295687091, + "objective/train/tokens_used": 100741600, + "theoretical_loss": 4.922125295687091, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927683049147443, + "loss": 3.5733, + "theoretical_loss": 4.922125295687091, + "tokens_seen": 80281600 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927582748244734, + "loss": 3.396, + "theoretical_loss": 4.92155114307519, + "tokens_seen": 80347136 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927482447342026, + "loss": 3.6166, + "theoretical_loss": 4.9209775895921375, + "tokens_seen": 80412672 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927382146439318, + "loss": 3.4859, + "theoretical_loss": 4.920404634125217, + "tokens_seen": 80478208 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492728184553661, + "loss": 3.2846, + "theoretical_loss": 4.919832275564684, + "tokens_seen": 80543744 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927181544633902, + "loss": 3.653, + "theoretical_loss": 4.9192605128037545, + "tokens_seen": 80609280 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004927081243731193, + "loss": 3.5658, + "theoretical_loss": 4.918689344738594, + "tokens_seen": 80674816 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926980942828485, + "loss": 3.5587, + "theoretical_loss": 4.918118770268305, + "tokens_seen": 80740352 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926880641925777, + "loss": 3.3881, + "theoretical_loss": 4.917548788294924, + "tokens_seen": 80805888 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926780341023069, + "loss": 3.4548, + "theoretical_loss": 4.916979397723403, + "tokens_seen": 80871424 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926680040120361, + "loss": 3.6868, + "theoretical_loss": 4.916410597461606, + "tokens_seen": 80936960 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926579739217654, + "loss": 3.6518, + "theoretical_loss": 4.9158423864202945, + "tokens_seen": 81002496 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926479438314944, + "loss": 3.4168, + "theoretical_loss": 4.915274763513119, + "tokens_seen": 81068032 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926379137412237, + "loss": 3.353, + "theoretical_loss": 4.914707727656609, + "tokens_seen": 81133568 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926278836509528, + "loss": 3.3831, + "theoretical_loss": 4.914141277770164, + "tokens_seen": 81199104 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926178535606821, + "loss": 3.1851, + "theoretical_loss": 4.913575412776042, + "tokens_seen": 81264640 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004926078234704113, + "loss": 3.7187, + "theoretical_loss": 4.91301013159935, + "tokens_seen": 81330176 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925977933801404, + "loss": 3.5052, + "theoretical_loss": 4.912445433168036, + "tokens_seen": 81395712 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925877632898696, + "loss": 3.5131, + "theoretical_loss": 4.911881316412876, + "tokens_seen": 81461248 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925777331995988, + "loss": 3.7694, + "theoretical_loss": 4.911317780267467, + "tokens_seen": 81526784 + }, + { + "epoch": 0.02, + "learning_rate": 0.000492567703109328, + "loss": 3.3832, + "theoretical_loss": 4.910754823668215, + "tokens_seen": 81592320 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925576730190572, + "loss": 3.8546, + "theoretical_loss": 4.910192445554329, + "tokens_seen": 81657856 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925476429287864, + "loss": 3.6127, + "theoretical_loss": 4.909630644867807, + "tokens_seen": 81723392 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925376128385155, + "loss": 3.3545, + "theoretical_loss": 4.909069420553429, + "tokens_seen": 81788928 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925275827482447, + "loss": 3.5589, + "theoretical_loss": 4.908508771558748, + "tokens_seen": 81854464 + }, + { + "epoch": 0.02, + "objective/train/docs_used": 57069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6827940940856934, + "objective/train/theoretical_loss": 4.907948696834077, + "objective/train/tokens_used": 102380000, + "theoretical_loss": 4.907948696834077, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925175526579739, + "loss": 3.7195, + "theoretical_loss": 4.907948696834077, + "tokens_seen": 81920000 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004925075225677031, + "loss": 3.5548, + "theoretical_loss": 4.907389195332484, + "tokens_seen": 81985536 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924974924774323, + "loss": 3.807, + "theoretical_loss": 4.906830266009781, + "tokens_seen": 82051072 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924874623871615, + "loss": 3.4511, + "theoretical_loss": 4.90627190782451, + "tokens_seen": 82116608 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924774322968907, + "loss": 3.7151, + "theoretical_loss": 4.905714119737945, + "tokens_seen": 82182144 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924674022066199, + "loss": 3.5237, + "theoretical_loss": 4.9051569007140685, + "tokens_seen": 82247680 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924573721163491, + "loss": 3.332, + "theoretical_loss": 4.904600249719573, + "tokens_seen": 82313216 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924473420260783, + "loss": 3.504, + "theoretical_loss": 4.90404416572385, + "tokens_seen": 82378752 + }, + { + "epoch": 0.02, + "learning_rate": 0.0004924373119358075, + "loss": 3.7408, + "theoretical_loss": 4.903488647698972, + "tokens_seen": 82444288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924272818455366, + "loss": 3.334, + "theoretical_loss": 4.902933694619698, + "tokens_seen": 82509824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004924172517552658, + "loss": 3.2703, + "theoretical_loss": 4.902379305463451, + "tokens_seen": 82575360 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492407221664995, + "loss": 3.5645, + "theoretical_loss": 4.901825479210317, + "tokens_seen": 82640896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923971915747242, + "loss": 3.3278, + "theoretical_loss": 4.9012722148430345, + "tokens_seen": 82706432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923871614844534, + "loss": 3.6277, + "theoretical_loss": 4.900719511346983, + "tokens_seen": 82771968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923771313941825, + "loss": 3.6241, + "theoretical_loss": 4.900167367710176, + "tokens_seen": 82837504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923671013039117, + "loss": 3.5454, + "theoretical_loss": 4.899615782923251, + "tokens_seen": 82903040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923570712136409, + "loss": 3.6864, + "theoretical_loss": 4.899064755979464, + "tokens_seen": 82968576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923470411233702, + "loss": 3.3855, + "theoretical_loss": 4.898514285874674, + "tokens_seen": 83034112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923370110330993, + "loss": 3.4924, + "theoretical_loss": 4.897964371607341, + "tokens_seen": 83099648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923269809428286, + "loss": 3.4934, + "theoretical_loss": 4.897415012178515, + "tokens_seen": 83165184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923169508525576, + "loss": 3.3992, + "theoretical_loss": 4.896866206591823, + "tokens_seen": 83230720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004923069207622869, + "loss": 3.5205, + "theoretical_loss": 4.896317953853467, + "tokens_seen": 83296256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922968906720161, + "loss": 3.624, + "theoretical_loss": 4.8957702529722145, + "tokens_seen": 83361792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922868605817453, + "loss": 3.5903, + "theoretical_loss": 4.895223102959381, + "tokens_seen": 83427328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922768304914745, + "loss": 3.7871, + "theoretical_loss": 4.894676502828836, + "tokens_seen": 83492864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 57750, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.804002046585083, + "objective/train/theoretical_loss": 4.894130451596979, + "objective/train/tokens_used": 104018400, + "theoretical_loss": 4.894130451596979, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922668004012036, + "loss": 3.4882, + "theoretical_loss": 4.894130451596979, + "tokens_seen": 83558400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922567703109328, + "loss": 3.2586, + "theoretical_loss": 4.893584948282745, + "tokens_seen": 83623936 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492246740220662, + "loss": 3.5408, + "theoretical_loss": 4.893039991907587, + "tokens_seen": 83689472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922367101303912, + "loss": 3.6106, + "theoretical_loss": 4.892495581495468, + "tokens_seen": 83755008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922266800401204, + "loss": 3.2746, + "theoretical_loss": 4.8919517160728585, + "tokens_seen": 83820544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922166499498495, + "loss": 3.5637, + "theoretical_loss": 4.891408394668723, + "tokens_seen": 83886080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004922066198595787, + "loss": 3.5945, + "theoretical_loss": 4.89086561631451, + "tokens_seen": 83951616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921965897693079, + "loss": 3.6297, + "theoretical_loss": 4.890323380044153, + "tokens_seen": 84017152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921865596790371, + "loss": 3.6036, + "theoretical_loss": 4.889781684894051, + "tokens_seen": 84082688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921765295887663, + "loss": 3.7009, + "theoretical_loss": 4.889240529903066, + "tokens_seen": 84148224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921664994984956, + "loss": 3.4759, + "theoretical_loss": 4.888699914112516, + "tokens_seen": 84213760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921564694082246, + "loss": 3.4833, + "theoretical_loss": 4.888159836566162, + "tokens_seen": 84279296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921464393179539, + "loss": 3.7335, + "theoretical_loss": 4.887620296310205, + "tokens_seen": 84344832 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492136409227683, + "loss": 3.8101, + "theoretical_loss": 4.887081292393276, + "tokens_seen": 84410368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921263791374123, + "loss": 3.6913, + "theoretical_loss": 4.886542823866424, + "tokens_seen": 84475904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921163490471415, + "loss": 3.5273, + "theoretical_loss": 4.886004889783116, + "tokens_seen": 84541440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004921063189568706, + "loss": 3.5718, + "theoretical_loss": 4.885467489199222, + "tokens_seen": 84606976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920962888665998, + "loss": 3.6503, + "theoretical_loss": 4.884930621173009, + "tokens_seen": 84672512 + }, + { + "epoch": 0.03, + "learning_rate": 0.000492086258776329, + "loss": 3.5557, + "theoretical_loss": 4.884394284765135, + "tokens_seen": 84738048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920762286860582, + "loss": 3.5114, + "theoretical_loss": 4.88385847903864, + "tokens_seen": 84803584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920661985957874, + "loss": 3.8069, + "theoretical_loss": 4.883323203058936, + "tokens_seen": 84869120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920561685055166, + "loss": 3.529, + "theoretical_loss": 4.882788455893802, + "tokens_seen": 84934656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920461384152457, + "loss": 3.5584, + "theoretical_loss": 4.882254236613376, + "tokens_seen": 85000192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920361083249749, + "loss": 3.7302, + "theoretical_loss": 4.881720544290144, + "tokens_seen": 85065728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920260782347041, + "loss": 3.4352, + "theoretical_loss": 4.881187377998938, + "tokens_seen": 85131264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 58904, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6649792194366455, + "objective/train/theoretical_loss": 4.880654736816922, + "objective/train/tokens_used": 105656800, + "theoretical_loss": 4.880654736816922, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920160481444333, + "loss": 3.7428, + "theoretical_loss": 4.880654736816922, + "tokens_seen": 85196800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004920060180541625, + "loss": 3.3637, + "theoretical_loss": 4.880122619823589, + "tokens_seen": 85262336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919959879638916, + "loss": 3.4179, + "theoretical_loss": 4.87959102610075, + "tokens_seen": 85327872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919859578736209, + "loss": 3.3888, + "theoretical_loss": 4.879059954732529, + "tokens_seen": 85393408 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049197592778335, + "loss": 3.7427, + "theoretical_loss": 4.878529404805356, + "tokens_seen": 85458944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919658976930793, + "loss": 3.7709, + "theoretical_loss": 4.8779993754079545, + "tokens_seen": 85524480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919558676028084, + "loss": 3.5494, + "theoretical_loss": 4.877469865631338, + "tokens_seen": 85590016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919458375125377, + "loss": 3.5295, + "theoretical_loss": 4.876940874568804, + "tokens_seen": 85655552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919358074222668, + "loss": 3.5878, + "theoretical_loss": 4.876412401315923, + "tokens_seen": 85721088 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491925777331996, + "loss": 3.57, + "theoretical_loss": 4.8758844449705325, + "tokens_seen": 85786624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919157472417252, + "loss": 3.5246, + "theoretical_loss": 4.875357004632728, + "tokens_seen": 85852160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004919057171514544, + "loss": 3.3963, + "theoretical_loss": 4.87483007940486, + "tokens_seen": 85917696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918956870611836, + "loss": 3.6586, + "theoretical_loss": 4.874303668391521, + "tokens_seen": 85983232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918856569709127, + "loss": 3.6073, + "theoretical_loss": 4.873777770699542, + "tokens_seen": 86048768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918756268806419, + "loss": 3.5692, + "theoretical_loss": 4.873252385437985, + "tokens_seen": 86114304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918655967903711, + "loss": 3.5077, + "theoretical_loss": 4.872727511718134, + "tokens_seen": 86179840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918555667001003, + "loss": 3.6584, + "theoretical_loss": 4.8722031486534885, + "tokens_seen": 86245376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918455366098295, + "loss": 3.4986, + "theoretical_loss": 4.8716792953597565, + "tokens_seen": 86310912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918355065195586, + "loss": 3.7466, + "theoretical_loss": 4.871155950954847, + "tokens_seen": 86376448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918254764292878, + "loss": 3.627, + "theoretical_loss": 4.870633114558865, + "tokens_seen": 86441984 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491815446339017, + "loss": 3.5282, + "theoretical_loss": 4.8701107852941, + "tokens_seen": 86507520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004918054162487463, + "loss": 3.1841, + "theoretical_loss": 4.869588962285023, + "tokens_seen": 86573056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917953861584754, + "loss": 3.6871, + "theoretical_loss": 4.869067644658276, + "tokens_seen": 86638592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917853560682047, + "loss": 3.6656, + "theoretical_loss": 4.86854683154267, + "tokens_seen": 86704128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917753259779337, + "loss": 3.5156, + "theoretical_loss": 4.868026522069172, + "tokens_seen": 86769664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 59520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.652179718017578, + "objective/train/theoretical_loss": 4.867506715370903, + "objective/train/tokens_used": 107295200, + "theoretical_loss": 4.867506715370903, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491765295887663, + "loss": 3.4928, + "theoretical_loss": 4.867506715370903, + "tokens_seen": 86835200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917552657973922, + "loss": 3.4802, + "theoretical_loss": 4.866987410583125, + "tokens_seen": 86900736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917452357071214, + "loss": 3.5752, + "theoretical_loss": 4.8664686068432434, + "tokens_seen": 86966272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917352056168506, + "loss": 3.5258, + "theoretical_loss": 4.8659503032907905, + "tokens_seen": 87031808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917251755265797, + "loss": 3.7263, + "theoretical_loss": 4.865432499067425, + "tokens_seen": 87097344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917151454363089, + "loss": 3.7067, + "theoretical_loss": 4.864915193316922, + "tokens_seen": 87162880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004917051153460381, + "loss": 3.4247, + "theoretical_loss": 4.864398385185169, + "tokens_seen": 87228416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916950852557673, + "loss": 3.5719, + "theoretical_loss": 4.863882073820154, + "tokens_seen": 87293952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916850551654965, + "loss": 3.5349, + "theoretical_loss": 4.863366258371963, + "tokens_seen": 87359488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916750250752258, + "loss": 3.5815, + "theoretical_loss": 4.862850937992777, + "tokens_seen": 87425024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916649949849548, + "loss": 3.5233, + "theoretical_loss": 4.862336111836852, + "tokens_seen": 87490560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916549648946841, + "loss": 3.5181, + "theoretical_loss": 4.861821779060529, + "tokens_seen": 87556096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916449348044132, + "loss": 3.369, + "theoretical_loss": 4.861307938822215, + "tokens_seen": 87621632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916349047141425, + "loss": 3.3532, + "theoretical_loss": 4.86079459028238, + "tokens_seen": 87687168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916248746238717, + "loss": 3.4634, + "theoretical_loss": 4.860281732603555, + "tokens_seen": 87752704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004916148445336008, + "loss": 3.64, + "theoretical_loss": 4.859769364950319, + "tokens_seen": 87818240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049160481444333, + "loss": 3.5164, + "theoretical_loss": 4.859257486489293, + "tokens_seen": 87883776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915947843530592, + "loss": 3.5639, + "theoretical_loss": 4.858746096389139, + "tokens_seen": 87949312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915847542627884, + "loss": 3.4632, + "theoretical_loss": 4.8582351938205495, + "tokens_seen": 88014848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915747241725176, + "loss": 3.2569, + "theoretical_loss": 4.857724777956237, + "tokens_seen": 88080384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915646940822468, + "loss": 3.4019, + "theoretical_loss": 4.857214847970939, + "tokens_seen": 88145920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915546639919759, + "loss": 3.266, + "theoretical_loss": 4.856705403041397, + "tokens_seen": 88211456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915446339017051, + "loss": 3.7935, + "theoretical_loss": 4.8561964423463655, + "tokens_seen": 88276992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915346038114343, + "loss": 3.7385, + "theoretical_loss": 4.8556879650665925, + "tokens_seen": 88342528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915245737211635, + "loss": 3.3908, + "theoretical_loss": 4.855179970384819, + "tokens_seen": 88408064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 60543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.110377788543701, + "objective/train/theoretical_loss": 4.8546724574857745, + "objective/train/tokens_used": 108933600, + "theoretical_loss": 4.8546724574857745, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915145436308927, + "loss": 3.4761, + "theoretical_loss": 4.8546724574857745, + "tokens_seen": 88473600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004915045135406218, + "loss": 3.442, + "theoretical_loss": 4.854165425556166, + "tokens_seen": 88539136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914944834503511, + "loss": 3.613, + "theoretical_loss": 4.853658873784675, + "tokens_seen": 88604672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914844533600802, + "loss": 3.7501, + "theoretical_loss": 4.853152801361952, + "tokens_seen": 88670208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914744232698095, + "loss": 3.4899, + "theoretical_loss": 4.852647207480604, + "tokens_seen": 88735744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914643931795386, + "loss": 3.6797, + "theoretical_loss": 4.8521420913352, + "tokens_seen": 88801280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914543630892679, + "loss": 3.2736, + "theoretical_loss": 4.851637452122251, + "tokens_seen": 88866816 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491444332998997, + "loss": 3.5697, + "theoretical_loss": 4.851133289040214, + "tokens_seen": 88932352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914343029087262, + "loss": 3.6628, + "theoretical_loss": 4.850629601289482, + "tokens_seen": 88997888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914242728184554, + "loss": 3.4603, + "theoretical_loss": 4.850126388072379, + "tokens_seen": 89063424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914142427281846, + "loss": 3.5286, + "theoretical_loss": 4.849623648593151, + "tokens_seen": 89128960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004914042126379138, + "loss": 3.4864, + "theoretical_loss": 4.849121382057964, + "tokens_seen": 89194496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913941825476429, + "loss": 3.5269, + "theoretical_loss": 4.848619587674898, + "tokens_seen": 89260032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913841524573721, + "loss": 3.4857, + "theoretical_loss": 4.8481182646539365, + "tokens_seen": 89325568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913741223671013, + "loss": 3.6437, + "theoretical_loss": 4.847617412206963, + "tokens_seen": 89391104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913640922768305, + "loss": 3.4963, + "theoretical_loss": 4.847117029547758, + "tokens_seen": 89456640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913540621865597, + "loss": 3.5452, + "theoretical_loss": 4.846617115891988, + "tokens_seen": 89522176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913440320962888, + "loss": 3.2749, + "theoretical_loss": 4.8461176704572, + "tokens_seen": 89587712 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491334002006018, + "loss": 3.319, + "theoretical_loss": 4.845618692462826, + "tokens_seen": 89653248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913239719157472, + "loss": 3.4855, + "theoretical_loss": 4.845120181130158, + "tokens_seen": 89718784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913139418254765, + "loss": 3.7812, + "theoretical_loss": 4.844622135682359, + "tokens_seen": 89784320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004913039117352056, + "loss": 3.1217, + "theoretical_loss": 4.84412455534445, + "tokens_seen": 89849856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912938816449349, + "loss": 3.3833, + "theoretical_loss": 4.843627439343306, + "tokens_seen": 89915392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912838515546639, + "loss": 3.5363, + "theoretical_loss": 4.843130786907647, + "tokens_seen": 89980928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912738214643932, + "loss": 3.4395, + "theoretical_loss": 4.842634597268036, + "tokens_seen": 90046464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 60961, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3933932781219482, + "objective/train/theoretical_loss": 4.842138869656871, + "objective/train/tokens_used": 110572000, + "theoretical_loss": 4.842138869656871, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912637913741224, + "loss": 3.5361, + "theoretical_loss": 4.842138869656871, + "tokens_seen": 90112000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912537612838516, + "loss": 3.1717, + "theoretical_loss": 4.841643603308384, + "tokens_seen": 90177536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912437311935808, + "loss": 3.6274, + "theoretical_loss": 4.841148797458627, + "tokens_seen": 90243072 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049123370110331, + "loss": 3.5958, + "theoretical_loss": 4.840654451345472, + "tokens_seen": 90308608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912236710130391, + "loss": 3.6702, + "theoretical_loss": 4.840160564208606, + "tokens_seen": 90374144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912136409227683, + "loss": 3.4133, + "theoretical_loss": 4.839667135289522, + "tokens_seen": 90439680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004912036108324975, + "loss": 3.3868, + "theoretical_loss": 4.839174163831515, + "tokens_seen": 90505216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911935807422267, + "loss": 3.4971, + "theoretical_loss": 4.838681649079677, + "tokens_seen": 90570752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911835506519559, + "loss": 3.581, + "theoretical_loss": 4.838189590280891, + "tokens_seen": 90636288 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491173520561685, + "loss": 3.5547, + "theoretical_loss": 4.837697986683825, + "tokens_seen": 90701824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911634904714142, + "loss": 3.6512, + "theoretical_loss": 4.837206837538925, + "tokens_seen": 90767360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911534603811434, + "loss": 3.189, + "theoretical_loss": 4.836716142098417, + "tokens_seen": 90832896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911434302908726, + "loss": 3.4003, + "theoretical_loss": 4.836225899616288, + "tokens_seen": 90898432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911334002006019, + "loss": 3.5343, + "theoretical_loss": 4.835736109348293, + "tokens_seen": 90963968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911233701103309, + "loss": 3.491, + "theoretical_loss": 4.8352467705519455, + "tokens_seen": 91029504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911133400200602, + "loss": 3.543, + "theoretical_loss": 4.83475788248651, + "tokens_seen": 91095040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004911033099297893, + "loss": 3.5829, + "theoretical_loss": 4.834269444412996, + "tokens_seen": 91160576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910932798395186, + "loss": 3.5038, + "theoretical_loss": 4.83378145559416, + "tokens_seen": 91226112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910832497492478, + "loss": 3.6003, + "theoretical_loss": 4.83329391529449, + "tokens_seen": 91291648 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491073219658977, + "loss": 3.5657, + "theoretical_loss": 4.832806822780208, + "tokens_seen": 91357184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910631895687061, + "loss": 3.3872, + "theoretical_loss": 4.832320177319261, + "tokens_seen": 91422720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910531594784353, + "loss": 3.4167, + "theoretical_loss": 4.831833978181314, + "tokens_seen": 91488256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910431293881645, + "loss": 3.2479, + "theoretical_loss": 4.831348224637752, + "tokens_seen": 91553792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910330992978937, + "loss": 3.5555, + "theoretical_loss": 4.830862915961665, + "tokens_seen": 91619328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910230692076229, + "loss": 3.4961, + "theoretical_loss": 4.830378051427852, + "tokens_seen": 91684864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 62148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0938072204589844, + "objective/train/theoretical_loss": 4.829893630312808, + "objective/train/tokens_used": 112210400, + "theoretical_loss": 4.829893630312808, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000491013039117352, + "loss": 3.1482, + "theoretical_loss": 4.829893630312808, + "tokens_seen": 91750400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004910030090270812, + "loss": 3.4885, + "theoretical_loss": 4.829409651894725, + "tokens_seen": 91815936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909929789368104, + "loss": 3.5385, + "theoretical_loss": 4.8289261154534815, + "tokens_seen": 91881472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909829488465397, + "loss": 3.2172, + "theoretical_loss": 4.828443020270642, + "tokens_seen": 91947008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909729187562688, + "loss": 3.351, + "theoretical_loss": 4.827960365629449, + "tokens_seen": 92012544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909628886659981, + "loss": 3.6262, + "theoretical_loss": 4.827478150814821, + "tokens_seen": 92078080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909528585757272, + "loss": 3.4536, + "theoretical_loss": 4.826996375113339, + "tokens_seen": 92143616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909428284854564, + "loss": 3.4569, + "theoretical_loss": 4.826515037813255, + "tokens_seen": 92209152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909327983951856, + "loss": 3.7922, + "theoretical_loss": 4.826034138204474, + "tokens_seen": 92274688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909227683049148, + "loss": 3.4368, + "theoretical_loss": 4.825553675578557, + "tokens_seen": 92340224 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490912738214644, + "loss": 3.3922, + "theoretical_loss": 4.825073649228713, + "tokens_seen": 92405760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004909027081243731, + "loss": 3.5078, + "theoretical_loss": 4.824594058449794, + "tokens_seen": 92471296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908926780341023, + "loss": 3.3106, + "theoretical_loss": 4.824114902538292, + "tokens_seen": 92536832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908826479438315, + "loss": 3.136, + "theoretical_loss": 4.823636180792329, + "tokens_seen": 92602368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908726178535607, + "loss": 3.3286, + "theoretical_loss": 4.823157892511661, + "tokens_seen": 92667904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908625877632899, + "loss": 3.6266, + "theoretical_loss": 4.82268003699766, + "tokens_seen": 92733440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490852557673019, + "loss": 3.5695, + "theoretical_loss": 4.8222026135533245, + "tokens_seen": 92798976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908425275827482, + "loss": 3.487, + "theoretical_loss": 4.821725621483262, + "tokens_seen": 92864512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908324974924774, + "loss": 3.503, + "theoretical_loss": 4.821249060093692, + "tokens_seen": 92930048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908224674022067, + "loss": 3.3758, + "theoretical_loss": 4.820772928692433, + "tokens_seen": 92995584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908124373119358, + "loss": 3.4497, + "theoretical_loss": 4.820297226588908, + "tokens_seen": 93061120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004908024072216651, + "loss": 3.6942, + "theoretical_loss": 4.819821953094133, + "tokens_seen": 93126656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907923771313941, + "loss": 3.6533, + "theoretical_loss": 4.819347107520713, + "tokens_seen": 93192192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907823470411234, + "loss": 3.3973, + "theoretical_loss": 4.818872689182836, + "tokens_seen": 93257728 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907723169508526, + "loss": 3.2205, + "theoretical_loss": 4.818398697396274, + "tokens_seen": 93323264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 62815, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5069005489349365, + "objective/train/theoretical_loss": 4.817925131478374, + "objective/train/tokens_used": 113848800, + "theoretical_loss": 4.817925131478374, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907622868605818, + "loss": 3.5578, + "theoretical_loss": 4.817925131478374, + "tokens_seen": 93388800 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490752256770311, + "loss": 3.5471, + "theoretical_loss": 4.817451990748047, + "tokens_seen": 93454336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907422266800401, + "loss": 3.6923, + "theoretical_loss": 4.81697927452578, + "tokens_seen": 93519872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907321965897693, + "loss": 3.5227, + "theoretical_loss": 4.816506982133613, + "tokens_seen": 93585408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907221664994985, + "loss": 3.2295, + "theoretical_loss": 4.816035112895147, + "tokens_seen": 93650944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907121364092277, + "loss": 3.1312, + "theoretical_loss": 4.815563666135532, + "tokens_seen": 93716480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004907021063189569, + "loss": 3.6798, + "theoretical_loss": 4.815092641181467, + "tokens_seen": 93782016 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490692076228686, + "loss": 3.7669, + "theoretical_loss": 4.814622037361195, + "tokens_seen": 93847552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906820461384152, + "loss": 3.6417, + "theoretical_loss": 4.814151854004493, + "tokens_seen": 93913088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906720160481444, + "loss": 3.2937, + "theoretical_loss": 4.813682090442674, + "tokens_seen": 93978624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906619859578736, + "loss": 3.2833, + "theoretical_loss": 4.81321274600858, + "tokens_seen": 94044160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906519558676028, + "loss": 3.2788, + "theoretical_loss": 4.812743820036577, + "tokens_seen": 94109696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906419257773321, + "loss": 3.5618, + "theoretical_loss": 4.81227531186255, + "tokens_seen": 94175232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906318956870611, + "loss": 3.5329, + "theoretical_loss": 4.811807220823899, + "tokens_seen": 94240768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906218655967904, + "loss": 3.6156, + "theoretical_loss": 4.811339546259536, + "tokens_seen": 94306304 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906118355065195, + "loss": 3.3777, + "theoretical_loss": 4.810872287509877, + "tokens_seen": 94371840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004906018054162488, + "loss": 3.5883, + "theoretical_loss": 4.810405443916842, + "tokens_seen": 94437376 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490591775325978, + "loss": 3.6907, + "theoretical_loss": 4.809939014823849, + "tokens_seen": 94502912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905817452357072, + "loss": 3.6602, + "theoretical_loss": 4.809472999575805, + "tokens_seen": 94568448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905717151454363, + "loss": 3.6322, + "theoretical_loss": 4.809007397519108, + "tokens_seen": 94633984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905616850551655, + "loss": 3.4204, + "theoretical_loss": 4.80854220800164, + "tokens_seen": 94699520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905516549648947, + "loss": 3.5405, + "theoretical_loss": 4.808077430372762, + "tokens_seen": 94765056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905416248746239, + "loss": 3.4647, + "theoretical_loss": 4.807613063983311, + "tokens_seen": 94830592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905315947843531, + "loss": 3.7424, + "theoretical_loss": 4.807149108185593, + "tokens_seen": 94896128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905215646940822, + "loss": 3.3372, + "theoretical_loss": 4.806685562333383, + "tokens_seen": 94961664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 64162, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0013229846954346, + "objective/train/theoretical_loss": 4.806222425781916, + "objective/train/tokens_used": 115487200, + "theoretical_loss": 4.806222425781916, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905115346038114, + "loss": 3.4672, + "theoretical_loss": 4.806222425781916, + "tokens_seen": 95027200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004905015045135406, + "loss": 3.4124, + "theoretical_loss": 4.805759697887886, + "tokens_seen": 95092736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904914744232698, + "loss": 3.5337, + "theoretical_loss": 4.80529737800944, + "tokens_seen": 95158272 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490481444332999, + "loss": 3.1987, + "theoretical_loss": 4.804835465506176, + "tokens_seen": 95223808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904714142427281, + "loss": 3.3264, + "theoretical_loss": 4.8043739597391335, + "tokens_seen": 95289344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904613841524574, + "loss": 3.421, + "theoretical_loss": 4.803912860070795, + "tokens_seen": 95354880 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904513540621865, + "loss": 3.5238, + "theoretical_loss": 4.803452165865082, + "tokens_seen": 95420416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904413239719158, + "loss": 3.4263, + "theoretical_loss": 4.802991876487344, + "tokens_seen": 95485952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904312938816449, + "loss": 3.3975, + "theoretical_loss": 4.802531991304359, + "tokens_seen": 95551488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904212637913742, + "loss": 3.5391, + "theoretical_loss": 4.802072509684331, + "tokens_seen": 95617024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904112337011033, + "loss": 3.4524, + "theoretical_loss": 4.801613430996884, + "tokens_seen": 95682560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004904012036108325, + "loss": 3.5944, + "theoretical_loss": 4.801154754613056, + "tokens_seen": 95748096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903911735205617, + "loss": 3.2372, + "theoretical_loss": 4.800696479905296, + "tokens_seen": 95813632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903811434302909, + "loss": 3.1815, + "theoretical_loss": 4.800238606247462, + "tokens_seen": 95879168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903711133400201, + "loss": 3.4315, + "theoretical_loss": 4.799781133014815, + "tokens_seen": 95944704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903610832497492, + "loss": 3.2851, + "theoretical_loss": 4.799324059584013, + "tokens_seen": 96010240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903510531594784, + "loss": 3.6757, + "theoretical_loss": 4.798867385333111, + "tokens_seen": 96075776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903410230692076, + "loss": 3.3841, + "theoretical_loss": 4.798411109641553, + "tokens_seen": 96141312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903309929789368, + "loss": 3.6343, + "theoretical_loss": 4.797955231890175, + "tokens_seen": 96206848 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490320962888666, + "loss": 3.3026, + "theoretical_loss": 4.797499751461188, + "tokens_seen": 96272384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903109327983952, + "loss": 3.3258, + "theoretical_loss": 4.797044667738188, + "tokens_seen": 96337920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004903009027081243, + "loss": 3.4414, + "theoretical_loss": 4.796589980106143, + "tokens_seen": 96403456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902908726178535, + "loss": 3.2512, + "theoretical_loss": 4.796135687951393, + "tokens_seen": 96468992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902808425275828, + "loss": 3.4072, + "theoretical_loss": 4.795681790661644, + "tokens_seen": 96534528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902708124373119, + "loss": 3.4574, + "theoretical_loss": 4.795228287625964, + "tokens_seen": 96600064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 64775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.715846538543701, + "objective/train/theoretical_loss": 4.794775178234782, + "objective/train/tokens_used": 117125600, + "theoretical_loss": 4.794775178234782, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902607823470412, + "loss": 3.622, + "theoretical_loss": 4.794775178234782, + "tokens_seen": 96665600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902507522567703, + "loss": 3.3135, + "theoretical_loss": 4.794322461879881, + "tokens_seen": 96731136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902407221664995, + "loss": 3.3367, + "theoretical_loss": 4.793870137954395, + "tokens_seen": 96796672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902306920762287, + "loss": 3.3923, + "theoretical_loss": 4.793418205852804, + "tokens_seen": 96862208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902206619859579, + "loss": 3.2944, + "theoretical_loss": 4.7929666649709315, + "tokens_seen": 96927744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902106318956871, + "loss": 3.4677, + "theoretical_loss": 4.7925155147059435, + "tokens_seen": 96993280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004902006018054163, + "loss": 3.5354, + "theoretical_loss": 4.792064754456337, + "tokens_seen": 97058816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901905717151454, + "loss": 3.4739, + "theoretical_loss": 4.791614383621943, + "tokens_seen": 97124352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901805416248746, + "loss": 3.4712, + "theoretical_loss": 4.79116440160392, + "tokens_seen": 97189888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901705115346038, + "loss": 3.6806, + "theoretical_loss": 4.790714807804751, + "tokens_seen": 97255424 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490160481444333, + "loss": 3.2497, + "theoretical_loss": 4.790265601628235, + "tokens_seen": 97320960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901504513540623, + "loss": 3.3692, + "theoretical_loss": 4.789816782479494, + "tokens_seen": 97386496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901404212637913, + "loss": 3.576, + "theoretical_loss": 4.789368349764956, + "tokens_seen": 97452032 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901303911735206, + "loss": 3.4743, + "theoretical_loss": 4.788920302892365, + "tokens_seen": 97517568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901203610832497, + "loss": 3.1291, + "theoretical_loss": 4.7884726412707606, + "tokens_seen": 97583104 + }, + { + "epoch": 0.03, + "learning_rate": 0.000490110330992979, + "loss": 3.5709, + "theoretical_loss": 4.788025364310492, + "tokens_seen": 97648640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004901003009027082, + "loss": 3.3566, + "theoretical_loss": 4.787578471423203, + "tokens_seen": 97714176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900902708124374, + "loss": 3.6399, + "theoretical_loss": 4.787131962021827, + "tokens_seen": 97779712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900802407221665, + "loss": 3.577, + "theoretical_loss": 4.786685835520595, + "tokens_seen": 97845248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900702106318957, + "loss": 3.4237, + "theoretical_loss": 4.786240091335019, + "tokens_seen": 97910784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900601805416249, + "loss": 3.3724, + "theoretical_loss": 4.785794728881895, + "tokens_seen": 97976320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900501504513541, + "loss": 3.3342, + "theoretical_loss": 4.785349747579299, + "tokens_seen": 98041856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900401203610833, + "loss": 3.5862, + "theoretical_loss": 4.784905146846581, + "tokens_seen": 98107392 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900300902708124, + "loss": 3.3465, + "theoretical_loss": 4.784460926104362, + "tokens_seen": 98172928 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900200601805416, + "loss": 3.3709, + "theoretical_loss": 4.7840170847745345, + "tokens_seen": 98238464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 65599, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.469925880432129, + "objective/train/theoretical_loss": 4.783573622280253, + "objective/train/tokens_used": 118764000, + "theoretical_loss": 4.783573622280253, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004900100300902708, + "loss": 3.5123, + "theoretical_loss": 4.783573622280253, + "tokens_seen": 98304000 + }, + { + "epoch": 0.03, + "learning_rate": 0.00049, + "loss": 3.1796, + "theoretical_loss": 4.7831305380459295, + "tokens_seen": 98369536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899899699097292, + "loss": 3.5071, + "theoretical_loss": 4.78268783149724, + "tokens_seen": 98435072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899799398194583, + "loss": 3.1618, + "theoretical_loss": 4.782245502061111, + "tokens_seen": 98500608 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899699097291876, + "loss": 3.5861, + "theoretical_loss": 4.781803549165718, + "tokens_seen": 98566144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899598796389167, + "loss": 3.7046, + "theoretical_loss": 4.781361972240484, + "tokens_seen": 98631680 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489949849548646, + "loss": 3.4213, + "theoretical_loss": 4.780920770716076, + "tokens_seen": 98697216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899398194583751, + "loss": 3.5335, + "theoretical_loss": 4.780479944024398, + "tokens_seen": 98762752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899297893681044, + "loss": 3.3783, + "theoretical_loss": 4.7800394915985915, + "tokens_seen": 98828288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899197592778335, + "loss": 3.5404, + "theoretical_loss": 4.779599412873031, + "tokens_seen": 98893824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004899097291875627, + "loss": 3.4117, + "theoretical_loss": 4.77915970728332, + "tokens_seen": 98959360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898996990972919, + "loss": 3.1311, + "theoretical_loss": 4.7787203742662845, + "tokens_seen": 99024896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898896690070211, + "loss": 3.3778, + "theoretical_loss": 4.778281413259977, + "tokens_seen": 99090432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898796389167503, + "loss": 3.3097, + "theoretical_loss": 4.777842823703665, + "tokens_seen": 99155968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898696088264794, + "loss": 3.4696, + "theoretical_loss": 4.777404605037833, + "tokens_seen": 99221504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898595787362086, + "loss": 3.5204, + "theoretical_loss": 4.7769667567041765, + "tokens_seen": 99287040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898495486459378, + "loss": 3.5602, + "theoretical_loss": 4.776529278145597, + "tokens_seen": 99352576 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489839518555667, + "loss": 3.4991, + "theoretical_loss": 4.776092168806207, + "tokens_seen": 99418112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898294884653962, + "loss": 3.6413, + "theoretical_loss": 4.775655428131313, + "tokens_seen": 99483648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898194583751254, + "loss": 3.6154, + "theoretical_loss": 4.775219055567424, + "tokens_seen": 99549184 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004898094282848545, + "loss": 3.4266, + "theoretical_loss": 4.7747830505622435, + "tokens_seen": 99614720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897993981945837, + "loss": 3.2347, + "theoretical_loss": 4.774347412564666, + "tokens_seen": 99680256 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489789368104313, + "loss": 3.5374, + "theoretical_loss": 4.773912141024771, + "tokens_seen": 99745792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897793380140421, + "loss": 3.3582, + "theoretical_loss": 4.773477235393826, + "tokens_seen": 99811328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897693079237714, + "loss": 3.4578, + "theoretical_loss": 4.773042695124279, + "tokens_seen": 99876864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 66410, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 4.062344551086426, + "objective/train/theoretical_loss": 4.772608519669756, + "objective/train/tokens_used": 120402400, + "theoretical_loss": 4.772608519669756, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897592778335005, + "loss": 3.5402, + "theoretical_loss": 4.772608519669756, + "tokens_seen": 99942400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897492477432297, + "loss": 3.5992, + "theoretical_loss": 4.772174708485057, + "tokens_seen": 100007936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897392176529589, + "loss": 3.4125, + "theoretical_loss": 4.771741261026155, + "tokens_seen": 100073472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897291875626881, + "loss": 3.3972, + "theoretical_loss": 4.771308176750187, + "tokens_seen": 100139008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897191574724173, + "loss": 3.7219, + "theoretical_loss": 4.77087545511546, + "tokens_seen": 100204544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004897091273821465, + "loss": 3.5976, + "theoretical_loss": 4.77044309558144, + "tokens_seen": 100270080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896990972918756, + "loss": 3.5781, + "theoretical_loss": 4.7700110976087515, + "tokens_seen": 100335616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896890672016048, + "loss": 3.4885, + "theoretical_loss": 4.769579460659173, + "tokens_seen": 100401152 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489679037111334, + "loss": 3.6787, + "theoretical_loss": 4.769148184195636, + "tokens_seen": 100466688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896690070210632, + "loss": 3.5478, + "theoretical_loss": 4.768717267682223, + "tokens_seen": 100532224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896589769307924, + "loss": 3.4238, + "theoretical_loss": 4.768286710584157, + "tokens_seen": 100597760 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896489468405215, + "loss": 3.3217, + "theoretical_loss": 4.767856512367805, + "tokens_seen": 100663296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896389167502507, + "loss": 3.3783, + "theoretical_loss": 4.767426672500677, + "tokens_seen": 100728832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896288866599799, + "loss": 3.2931, + "theoretical_loss": 4.766997190451413, + "tokens_seen": 100794368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896188565697091, + "loss": 3.6434, + "theoretical_loss": 4.766568065689789, + "tokens_seen": 100859904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004896088264794384, + "loss": 3.5874, + "theoretical_loss": 4.766139297686712, + "tokens_seen": 100925440 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895987963891674, + "loss": 3.4533, + "theoretical_loss": 4.76571088591421, + "tokens_seen": 100990976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895887662988967, + "loss": 3.6816, + "theoretical_loss": 4.7652828298454395, + "tokens_seen": 101056512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895787362086259, + "loss": 3.3644, + "theoretical_loss": 4.764855128954675, + "tokens_seen": 101122048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895687061183551, + "loss": 3.2458, + "theoretical_loss": 4.764427782717309, + "tokens_seen": 101187584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895586760280843, + "loss": 3.3975, + "theoretical_loss": 4.764000790609848, + "tokens_seen": 101253120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895486459378135, + "loss": 3.3918, + "theoretical_loss": 4.763574152109907, + "tokens_seen": 101318656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895386158475426, + "loss": 3.4413, + "theoretical_loss": 4.763147866696213, + "tokens_seen": 101384192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895285857572718, + "loss": 3.154, + "theoretical_loss": 4.762721933848593, + "tokens_seen": 101449728 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489518555667001, + "loss": 3.367, + "theoretical_loss": 4.762296353047979, + "tokens_seen": 101515264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 66745, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.258596897125244, + "objective/train/theoretical_loss": 4.7618711237764035, + "objective/train/tokens_used": 122040800, + "theoretical_loss": 4.7618711237764035, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004895085255767302, + "loss": 3.3147, + "theoretical_loss": 4.7618711237764035, + "tokens_seen": 101580800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894984954864594, + "loss": 3.2477, + "theoretical_loss": 4.761446245516988, + "tokens_seen": 101646336 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894884653961885, + "loss": 3.5315, + "theoretical_loss": 4.7610217177539536, + "tokens_seen": 101711872 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894784353059178, + "loss": 3.3191, + "theoretical_loss": 4.760597539972606, + "tokens_seen": 101777408 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894684052156469, + "loss": 3.0442, + "theoretical_loss": 4.760173711659341, + "tokens_seen": 101842944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894583751253762, + "loss": 3.4036, + "theoretical_loss": 4.759750232301635, + "tokens_seen": 101908480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894483450351053, + "loss": 3.3851, + "theoretical_loss": 4.759327101388048, + "tokens_seen": 101974016 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894383149448346, + "loss": 3.5575, + "theoretical_loss": 4.7589043184082165, + "tokens_seen": 102039552 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894282848545637, + "loss": 3.3678, + "theoretical_loss": 4.75848188285285, + "tokens_seen": 102105088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894182547642929, + "loss": 3.5807, + "theoretical_loss": 4.7580597942137315, + "tokens_seen": 102170624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004894082246740221, + "loss": 3.4755, + "theoretical_loss": 4.757638051983713, + "tokens_seen": 102236160 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893981945837513, + "loss": 3.308, + "theoretical_loss": 4.757216655656713, + "tokens_seen": 102301696 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893881644934805, + "loss": 3.4312, + "theoretical_loss": 4.756795604727708, + "tokens_seen": 102367232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893781344032096, + "loss": 3.4421, + "theoretical_loss": 4.7563748986927425, + "tokens_seen": 102432768 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893681043129388, + "loss": 3.3071, + "theoretical_loss": 4.755954537048913, + "tokens_seen": 102498304 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489358074222668, + "loss": 3.7184, + "theoretical_loss": 4.755534519294368, + "tokens_seen": 102563840 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893480441323972, + "loss": 3.6644, + "theoretical_loss": 4.7551148449283165, + "tokens_seen": 102629376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893380140421264, + "loss": 3.3233, + "theoretical_loss": 4.754695513451004, + "tokens_seen": 102694912 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893279839518556, + "loss": 3.6128, + "theoretical_loss": 4.754276524363733, + "tokens_seen": 102760448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893179538615847, + "loss": 3.4243, + "theoretical_loss": 4.75385787716884, + "tokens_seen": 102825984 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004893079237713139, + "loss": 3.6836, + "theoretical_loss": 4.7534395713697055, + "tokens_seen": 102891520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892978936810432, + "loss": 3.6634, + "theoretical_loss": 4.753021606470748, + "tokens_seen": 102957056 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892878635907723, + "loss": 3.3886, + "theoretical_loss": 4.752603981977417, + "tokens_seen": 103022592 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892778335005016, + "loss": 3.4605, + "theoretical_loss": 4.752186697396198, + "tokens_seen": 103088128 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892678034102307, + "loss": 3.3429, + "theoretical_loss": 4.7517697522345985, + "tokens_seen": 103153664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 68050, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.391291379928589, + "objective/train/theoretical_loss": 4.7513531460011595, + "objective/train/tokens_used": 123679200, + "theoretical_loss": 4.7513531460011595, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892577733199599, + "loss": 3.488, + "theoretical_loss": 4.7513531460011595, + "tokens_seen": 103219200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892477432296891, + "loss": 3.4412, + "theoretical_loss": 4.750936878205438, + "tokens_seen": 103284736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892377131394183, + "loss": 3.1551, + "theoretical_loss": 4.750520948358019, + "tokens_seen": 103350272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892276830491475, + "loss": 3.3288, + "theoretical_loss": 4.750105355970497, + "tokens_seen": 103415808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892176529588767, + "loss": 3.3249, + "theoretical_loss": 4.749690100555487, + "tokens_seen": 103481344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004892076228686058, + "loss": 3.4856, + "theoretical_loss": 4.749275181626612, + "tokens_seen": 103546880 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489197592778335, + "loss": 3.447, + "theoretical_loss": 4.748860598698509, + "tokens_seen": 103612416 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891875626880642, + "loss": 3.3712, + "theoretical_loss": 4.748446351286817, + "tokens_seen": 103677952 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891775325977934, + "loss": 3.3355, + "theoretical_loss": 4.748032438908181, + "tokens_seen": 103743488 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891675025075226, + "loss": 3.6823, + "theoretical_loss": 4.747618861080247, + "tokens_seen": 103809024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891574724172517, + "loss": 3.5451, + "theoretical_loss": 4.74720561732166, + "tokens_seen": 103874560 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891474423269809, + "loss": 3.3377, + "theoretical_loss": 4.746792707152057, + "tokens_seen": 103940096 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891374122367101, + "loss": 3.2533, + "theoretical_loss": 4.746380130092074, + "tokens_seen": 104005632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891273821464393, + "loss": 3.5114, + "theoretical_loss": 4.745967885663333, + "tokens_seen": 104071168 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891173520561686, + "loss": 3.5891, + "theoretical_loss": 4.745555973388447, + "tokens_seen": 104136704 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004891073219658976, + "loss": 3.5146, + "theoretical_loss": 4.745144392791008, + "tokens_seen": 104202240 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890972918756269, + "loss": 3.477, + "theoretical_loss": 4.744733143395598, + "tokens_seen": 104267776 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890872617853561, + "loss": 3.5138, + "theoretical_loss": 4.744322224727773, + "tokens_seen": 104333312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890772316950853, + "loss": 3.4684, + "theoretical_loss": 4.743911636314069, + "tokens_seen": 104398848 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890672016048145, + "loss": 3.3884, + "theoretical_loss": 4.743501377681995, + "tokens_seen": 104464384 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890571715145437, + "loss": 3.2946, + "theoretical_loss": 4.743091448360034, + "tokens_seen": 104529920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890471414242728, + "loss": 3.362, + "theoretical_loss": 4.742681847877636, + "tokens_seen": 104595456 + }, + { + "epoch": 0.03, + "learning_rate": 0.000489037111334002, + "loss": 3.1456, + "theoretical_loss": 4.742272575765218, + "tokens_seen": 104660992 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890270812437312, + "loss": 3.6056, + "theoretical_loss": 4.741863631554163, + "tokens_seen": 104726528 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890170511534604, + "loss": 3.5804, + "theoretical_loss": 4.741455014776813, + "tokens_seen": 104792064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 68417, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.796355724334717, + "objective/train/theoretical_loss": 4.741046724966468, + "objective/train/tokens_used": 125317600, + "theoretical_loss": 4.741046724966468, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004890070210631896, + "loss": 3.5708, + "theoretical_loss": 4.741046724966468, + "tokens_seen": 104857600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889969909729187, + "loss": 3.1131, + "theoretical_loss": 4.74063876165739, + "tokens_seen": 104923136 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889869608826479, + "loss": 3.4213, + "theoretical_loss": 4.740231124384789, + "tokens_seen": 104988672 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889769307923771, + "loss": 3.3977, + "theoretical_loss": 4.739823812684829, + "tokens_seen": 105054208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889669007021063, + "loss": 3.345, + "theoretical_loss": 4.739416826094621, + "tokens_seen": 105119744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889568706118355, + "loss": 3.4305, + "theoretical_loss": 4.739010164152224, + "tokens_seen": 105185280 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889468405215647, + "loss": 3.393, + "theoretical_loss": 4.7386038263966395, + "tokens_seen": 105250816 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889368104312939, + "loss": 3.4701, + "theoretical_loss": 4.73819781236781, + "tokens_seen": 105316352 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488926780341023, + "loss": 3.2626, + "theoretical_loss": 4.737792121606619, + "tokens_seen": 105381888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889167502507523, + "loss": 3.364, + "theoretical_loss": 4.737386753654883, + "tokens_seen": 105447424 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004889067201604815, + "loss": 3.4215, + "theoretical_loss": 4.736981708055353, + "tokens_seen": 105512960 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888966900702107, + "loss": 3.2655, + "theoretical_loss": 4.7365769843517125, + "tokens_seen": 105578496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888866599799398, + "loss": 3.2771, + "theoretical_loss": 4.736172582088571, + "tokens_seen": 105644032 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488876629889669, + "loss": 3.3345, + "theoretical_loss": 4.735768500811469, + "tokens_seen": 105709568 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888665997993982, + "loss": 3.4764, + "theoretical_loss": 4.735364740066865, + "tokens_seen": 105775104 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888565697091274, + "loss": 3.7094, + "theoretical_loss": 4.734961299402143, + "tokens_seen": 105840640 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888465396188566, + "loss": 3.4389, + "theoretical_loss": 4.734558178365605, + "tokens_seen": 105906176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888365095285858, + "loss": 3.5066, + "theoretical_loss": 4.734155376506466, + "tokens_seen": 105971712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888264794383149, + "loss": 3.2004, + "theoretical_loss": 4.733752893374863, + "tokens_seen": 106037248 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888164493480441, + "loss": 3.9504, + "theoretical_loss": 4.733350728521835, + "tokens_seen": 106102784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004888064192577733, + "loss": 3.3078, + "theoretical_loss": 4.732948881499336, + "tokens_seen": 106168320 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887963891675025, + "loss": 3.258, + "theoretical_loss": 4.7325473518602275, + "tokens_seen": 106233856 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887863590772317, + "loss": 3.4263, + "theoretical_loss": 4.732146139158271, + "tokens_seen": 106299392 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488776328986961, + "loss": 3.2168, + "theoretical_loss": 4.731745242948134, + "tokens_seen": 106364928 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048876629889669, + "loss": 3.7064, + "theoretical_loss": 4.731344662785381, + "tokens_seen": 106430464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 69666, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1660373210906982, + "objective/train/theoretical_loss": 4.730944398226477, + "objective/train/tokens_used": 126956000, + "theoretical_loss": 4.730944398226477, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887562688064193, + "loss": 3.4624, + "theoretical_loss": 4.730944398226477, + "tokens_seen": 106496000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887462387161484, + "loss": 3.2648, + "theoretical_loss": 4.730544448828777, + "tokens_seen": 106561536 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887362086258777, + "loss": 3.3377, + "theoretical_loss": 4.730144814150533, + "tokens_seen": 106627072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887261785356069, + "loss": 3.182, + "theoretical_loss": 4.729745493750885, + "tokens_seen": 106692608 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488716148445336, + "loss": 3.5011, + "theoretical_loss": 4.729346487189863, + "tokens_seen": 106758144 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004887061183550652, + "loss": 3.4765, + "theoretical_loss": 4.728947794028379, + "tokens_seen": 106823680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886960882647944, + "loss": 3.5698, + "theoretical_loss": 4.728549413828231, + "tokens_seen": 106889216 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886860581745236, + "loss": 3.5244, + "theoretical_loss": 4.728151346152098, + "tokens_seen": 106954752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886760280842528, + "loss": 3.2724, + "theoretical_loss": 4.727753590563535, + "tokens_seen": 107020288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886659979939819, + "loss": 3.2698, + "theoretical_loss": 4.727356146626976, + "tokens_seen": 107085824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886559679037111, + "loss": 3.3792, + "theoretical_loss": 4.726959013907727, + "tokens_seen": 107151360 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886459378134403, + "loss": 3.0486, + "theoretical_loss": 4.7265621919719685, + "tokens_seen": 107216896 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886359077231695, + "loss": 3.2127, + "theoretical_loss": 4.726165680386746, + "tokens_seen": 107282432 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886258776328988, + "loss": 3.4488, + "theoretical_loss": 4.725769478719978, + "tokens_seen": 107347968 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886158475426278, + "loss": 3.6296, + "theoretical_loss": 4.725373586540442, + "tokens_seen": 107413504 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004886058174523571, + "loss": 3.1497, + "theoretical_loss": 4.724978003417782, + "tokens_seen": 107479040 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885957873620863, + "loss": 3.0499, + "theoretical_loss": 4.724582728922501, + "tokens_seen": 107544576 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885857572718155, + "loss": 3.3971, + "theoretical_loss": 4.724187762625959, + "tokens_seen": 107610112 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885757271815447, + "loss": 3.4614, + "theoretical_loss": 4.723793104100376, + "tokens_seen": 107675648 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885656970912739, + "loss": 3.2867, + "theoretical_loss": 4.72339875291882, + "tokens_seen": 107741184 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488555667001003, + "loss": 3.5243, + "theoretical_loss": 4.723004708655214, + "tokens_seen": 107806720 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885456369107322, + "loss": 3.471, + "theoretical_loss": 4.722610970884331, + "tokens_seen": 107872256 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885356068204614, + "loss": 3.4896, + "theoretical_loss": 4.722217539181788, + "tokens_seen": 107937792 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885255767301906, + "loss": 3.5163, + "theoretical_loss": 4.7218244131240485, + "tokens_seen": 108003328 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004885155466399198, + "loss": 3.1836, + "theoretical_loss": 4.721431592288419, + "tokens_seen": 108068864 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 70233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.18623423576355, + "objective/train/theoretical_loss": 4.721039076253046, + "objective/train/tokens_used": 128594400, + "theoretical_loss": 4.721039076253046, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488505516549649, + "loss": 3.2707, + "theoretical_loss": 4.721039076253046, + "tokens_seen": 108134400 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884954864593781, + "loss": 3.3744, + "theoretical_loss": 4.720646864596914, + "tokens_seen": 108199936 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884854563691073, + "loss": 3.2403, + "theoretical_loss": 4.720254956899844, + "tokens_seen": 108265472 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884754262788365, + "loss": 3.4192, + "theoretical_loss": 4.719863352742491, + "tokens_seen": 108331008 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884653961885657, + "loss": 3.2221, + "theoretical_loss": 4.719472051706341, + "tokens_seen": 108396544 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884553660982949, + "loss": 3.4325, + "theoretical_loss": 4.71908105337371, + "tokens_seen": 108462080 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884453360080241, + "loss": 3.3533, + "theoretical_loss": 4.718690357327743, + "tokens_seen": 108527616 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884353059177532, + "loss": 3.5112, + "theoretical_loss": 4.718299963152407, + "tokens_seen": 108593152 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884252758274825, + "loss": 3.2922, + "theoretical_loss": 4.7179098704324955, + "tokens_seen": 108658688 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884152457372117, + "loss": 3.4903, + "theoretical_loss": 4.717520078753621, + "tokens_seen": 108724224 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004884052156469409, + "loss": 3.5257, + "theoretical_loss": 4.717130587702217, + "tokens_seen": 108789760 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048839518555667, + "loss": 3.371, + "theoretical_loss": 4.716741396865531, + "tokens_seen": 108855296 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883851554663992, + "loss": 3.388, + "theoretical_loss": 4.716352505831628, + "tokens_seen": 108920832 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883751253761284, + "loss": 3.5684, + "theoretical_loss": 4.715963914189384, + "tokens_seen": 108986368 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883650952858576, + "loss": 3.3996, + "theoretical_loss": 4.715575621528487, + "tokens_seen": 109051904 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883550651955868, + "loss": 3.1784, + "theoretical_loss": 4.71518762743943, + "tokens_seen": 109117440 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488345035105316, + "loss": 3.2919, + "theoretical_loss": 4.714799931513517, + "tokens_seen": 109182976 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883350050150451, + "loss": 3.1558, + "theoretical_loss": 4.714412533342855, + "tokens_seen": 109248512 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883249749247743, + "loss": 3.5333, + "theoretical_loss": 4.7140254325203506, + "tokens_seen": 109314048 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883149448345035, + "loss": 3.6226, + "theoretical_loss": 4.713638628639713, + "tokens_seen": 109379584 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004883049147442327, + "loss": 3.2969, + "theoretical_loss": 4.713252121295449, + "tokens_seen": 109445120 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882948846539619, + "loss": 3.6223, + "theoretical_loss": 4.712865910082863, + "tokens_seen": 109510656 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882848545636911, + "loss": 3.4854, + "theoretical_loss": 4.712479994598052, + "tokens_seen": 109576192 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882748244734203, + "loss": 3.4307, + "theoretical_loss": 4.7120943744379025, + "tokens_seen": 109641728 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048826479438314946, + "loss": 3.3526, + "theoretical_loss": 4.7117090492000955, + "tokens_seen": 109707264 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 70991, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.7265214920043945, + "objective/train/theoretical_loss": 4.711324018483099, + "objective/train/tokens_used": 130232800, + "theoretical_loss": 4.711324018483099, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048825476429287864, + "loss": 3.4221, + "theoretical_loss": 4.711324018483099, + "tokens_seen": 109772800 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882447342026078, + "loss": 3.156, + "theoretical_loss": 4.710939281886164, + "tokens_seen": 109838336 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488234704112337, + "loss": 3.5109, + "theoretical_loss": 4.71055483900933, + "tokens_seen": 109903872 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048822467402206624, + "loss": 3.6234, + "theoretical_loss": 4.710170689453415, + "tokens_seen": 109969408 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048821464393179536, + "loss": 3.3112, + "theoretical_loss": 4.709786832820017, + "tokens_seen": 110034944 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004882046138415246, + "loss": 3.3367, + "theoretical_loss": 4.7094032687115135, + "tokens_seen": 110100480 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881945837512537, + "loss": 3.3711, + "theoretical_loss": 4.709019996731058, + "tokens_seen": 110166016 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048818455366098296, + "loss": 3.1866, + "theoretical_loss": 4.708637016482579, + "tokens_seen": 110231552 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048817452357071214, + "loss": 3.3009, + "theoretical_loss": 4.708254327570774, + "tokens_seen": 110297088 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881644934804413, + "loss": 3.6798, + "theoretical_loss": 4.7078719296011124, + "tokens_seen": 110362624 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881544633901705, + "loss": 3.5188, + "theoretical_loss": 4.7074898221798325, + "tokens_seen": 110428160 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048814443329989974, + "loss": 3.1, + "theoretical_loss": 4.707108004913936, + "tokens_seen": 110493696 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048813440320962887, + "loss": 3.5579, + "theoretical_loss": 4.706726477411193, + "tokens_seen": 110559232 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004881243731193581, + "loss": 3.3834, + "theoretical_loss": 4.706345239280133, + "tokens_seen": 110624768 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048811434302908723, + "loss": 3.3456, + "theoretical_loss": 4.705964290130046, + "tokens_seen": 110690304 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048810431293881646, + "loss": 3.3642, + "theoretical_loss": 4.705583629570983, + "tokens_seen": 110755840 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048809428284854564, + "loss": 3.2147, + "theoretical_loss": 4.705203257213746, + "tokens_seen": 110821376 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880842527582748, + "loss": 3.4359, + "theoretical_loss": 4.7048231726699, + "tokens_seen": 110886912 + }, + { + "epoch": 0.03, + "learning_rate": 0.000488074222668004, + "loss": 3.163, + "theoretical_loss": 4.7044433755517545, + "tokens_seen": 110952448 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880641925777332, + "loss": 3.2166, + "theoretical_loss": 4.704063865472374, + "tokens_seen": 111017984 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048805416248746237, + "loss": 3.2815, + "theoretical_loss": 4.703684642045571, + "tokens_seen": 111083520 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004880441323971916, + "loss": 3.3855, + "theoretical_loss": 4.703305704885906, + "tokens_seen": 111149056 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048803410230692073, + "loss": 3.476, + "theoretical_loss": 4.7029270536086845, + "tokens_seen": 111214592 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048802407221664997, + "loss": 3.1811, + "theoretical_loss": 4.702548687829953, + "tokens_seen": 111280128 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048801404212637915, + "loss": 3.212, + "theoretical_loss": 4.702170607166503, + "tokens_seen": 111345664 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 71566, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1423094272613525, + "objective/train/theoretical_loss": 4.701792811235861, + "objective/train/tokens_used": 131871200, + "theoretical_loss": 4.701792811235861, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048800401203610833, + "loss": 3.3106, + "theoretical_loss": 4.701792811235861, + "tokens_seen": 111411200 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879939819458375, + "loss": 3.3804, + "theoretical_loss": 4.701415299656295, + "tokens_seen": 111476736 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879839518555667, + "loss": 3.3137, + "theoretical_loss": 4.701038072046808, + "tokens_seen": 111542272 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879739217652959, + "loss": 3.4184, + "theoretical_loss": 4.7006611280271375, + "tokens_seen": 111607808 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879638916750251, + "loss": 3.5134, + "theoretical_loss": 4.700284467217749, + "tokens_seen": 111673344 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879538615847543, + "loss": 3.515, + "theoretical_loss": 4.699908089239845, + "tokens_seen": 111738880 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048794383149448347, + "loss": 3.4391, + "theoretical_loss": 4.6995319937153495, + "tokens_seen": 111804416 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048793380140421265, + "loss": 3.4928, + "theoretical_loss": 4.699156180266918, + "tokens_seen": 111869952 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048792377131394183, + "loss": 3.4204, + "theoretical_loss": 4.69878064851793, + "tokens_seen": 111935488 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048791374122367107, + "loss": 3.2864, + "theoretical_loss": 4.698405398092486, + "tokens_seen": 112001024 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004879037111334002, + "loss": 3.52, + "theoretical_loss": 4.698030428615411, + "tokens_seen": 112066560 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048789368104312943, + "loss": 3.2593, + "theoretical_loss": 4.6976557397122445, + "tokens_seen": 112132096 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048788365095285856, + "loss": 3.3313, + "theoretical_loss": 4.697281331009249, + "tokens_seen": 112197632 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878736208625878, + "loss": 3.1676, + "theoretical_loss": 4.6969072021333975, + "tokens_seen": 112263168 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048786359077231697, + "loss": 3.4483, + "theoretical_loss": 4.696533352712381, + "tokens_seen": 112328704 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048785356068204615, + "loss": 3.3785, + "theoretical_loss": 4.696159782374602, + "tokens_seen": 112394240 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048784353059177533, + "loss": 3.0219, + "theoretical_loss": 4.695786490749171, + "tokens_seen": 112459776 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048783350050150457, + "loss": 3.3436, + "theoretical_loss": 4.6954134774659115, + "tokens_seen": 112525312 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004878234704112337, + "loss": 3.3403, + "theoretical_loss": 4.695040742155349, + "tokens_seen": 112590848 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048781344032096293, + "loss": 3.4199, + "theoretical_loss": 4.694668284448717, + "tokens_seen": 112656384 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048780341023069206, + "loss": 3.0947, + "theoretical_loss": 4.694296103977953, + "tokens_seen": 112721920 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877933801404213, + "loss": 2.9975, + "theoretical_loss": 4.693924200375694, + "tokens_seen": 112787456 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877833500501505, + "loss": 3.2263, + "theoretical_loss": 4.693552573275277, + "tokens_seen": 112852992 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048777331995987966, + "loss": 3.3911, + "theoretical_loss": 4.69318122231074, + "tokens_seen": 112918528 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048776328986960884, + "loss": 3.3579, + "theoretical_loss": 4.692810147116814, + "tokens_seen": 112984064 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 72808, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.543510913848877, + "objective/train/theoretical_loss": 4.692439347328926, + "objective/train/tokens_used": 133509600, + "theoretical_loss": 4.692439347328926, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487753259779338, + "loss": 3.5704, + "theoretical_loss": 4.692439347328926, + "tokens_seen": 113049600 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877432296890672, + "loss": 3.2623, + "theoretical_loss": 4.692068822583197, + "tokens_seen": 113115136 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048773319959879644, + "loss": 3.7086, + "theoretical_loss": 4.6916985725164375, + "tokens_seen": 113180672 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048772316950852556, + "loss": 3.2168, + "theoretical_loss": 4.691328596766148, + "tokens_seen": 113246208 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877131394182548, + "loss": 3.5721, + "theoretical_loss": 4.690958894970518, + "tokens_seen": 113311744 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004877031093279839, + "loss": 3.5405, + "theoretical_loss": 4.690589466768422, + "tokens_seen": 113377280 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048769307923771316, + "loss": 3.4867, + "theoretical_loss": 4.69022031179942, + "tokens_seen": 113442816 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048768304914744234, + "loss": 3.3823, + "theoretical_loss": 4.689851429703753, + "tokens_seen": 113508352 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876730190571715, + "loss": 3.242, + "theoretical_loss": 4.689482820122342, + "tokens_seen": 113573888 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876629889669007, + "loss": 3.6015, + "theoretical_loss": 4.689114482696793, + "tokens_seen": 113639424 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048765295887662994, + "loss": 3.4168, + "theoretical_loss": 4.6887464170693836, + "tokens_seen": 113704960 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048764292878635907, + "loss": 3.4562, + "theoretical_loss": 4.6883786228830715, + "tokens_seen": 113770496 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004876328986960883, + "loss": 3.1948, + "theoretical_loss": 4.688011099781486, + "tokens_seen": 113836032 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048762286860581743, + "loss": 3.2999, + "theoretical_loss": 4.687643847408929, + "tokens_seen": 113901568 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048761283851554666, + "loss": 3.342, + "theoretical_loss": 4.687276865410377, + "tokens_seen": 113967104 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048760280842527584, + "loss": 3.4757, + "theoretical_loss": 4.686910153431472, + "tokens_seen": 114032640 + }, + { + "epoch": 0.03, + "learning_rate": 0.000487592778335005, + "loss": 3.5307, + "theoretical_loss": 4.686543711118526, + "tokens_seen": 114098176 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875827482447342, + "loss": 3.2597, + "theoretical_loss": 4.686177538118516, + "tokens_seen": 114163712 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875727181544634, + "loss": 3.4569, + "theoretical_loss": 4.685811634079084, + "tokens_seen": 114229248 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048756268806419257, + "loss": 3.3531, + "theoretical_loss": 4.685445998648532, + "tokens_seen": 114294784 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875526579739218, + "loss": 3.4034, + "theoretical_loss": 4.685080631475831, + "tokens_seen": 114360320 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048754262788365093, + "loss": 3.0558, + "theoretical_loss": 4.6847155322106016, + "tokens_seen": 114425856 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048753259779338017, + "loss": 3.4359, + "theoretical_loss": 4.6843507005031295, + "tokens_seen": 114491392 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048752256770310935, + "loss": 3.4293, + "theoretical_loss": 4.683986136004355, + "tokens_seen": 114556928 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048751253761283853, + "loss": 3.1354, + "theoretical_loss": 4.6836218383658705, + "tokens_seen": 114622464 + }, + { + "epoch": 0.03, + "objective/train/docs_used": 73349, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8527109622955322, + "objective/train/theoretical_loss": 4.683257807239925, + "objective/train/tokens_used": 135148000, + "theoretical_loss": 4.683257807239925, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004875025075225677, + "loss": 3.1073, + "theoretical_loss": 4.683257807239925, + "tokens_seen": 114688000 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874924774322969, + "loss": 3.096, + "theoretical_loss": 4.682894042279417, + "tokens_seen": 114753536 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048748244734202607, + "loss": 3.2853, + "theoretical_loss": 4.682530543137897, + "tokens_seen": 114819072 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874724172517553, + "loss": 3.6298, + "theoretical_loss": 4.68216730946956, + "tokens_seen": 114884608 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048746238716148443, + "loss": 3.305, + "theoretical_loss": 4.6818043409292525, + "tokens_seen": 114950144 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048745235707121367, + "loss": 3.4946, + "theoretical_loss": 4.681441637172463, + "tokens_seen": 115015680 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874423269809428, + "loss": 3.3972, + "theoretical_loss": 4.681079197855325, + "tokens_seen": 115081216 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048743229689067203, + "loss": 3.1421, + "theoretical_loss": 4.680717022634612, + "tokens_seen": 115146752 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874222668004012, + "loss": 3.6661, + "theoretical_loss": 4.680355111167742, + "tokens_seen": 115212288 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874122367101304, + "loss": 3.2117, + "theoretical_loss": 4.679993463112769, + "tokens_seen": 115277824 + }, + { + "epoch": 0.03, + "learning_rate": 0.0004874022066198596, + "loss": 3.4083, + "theoretical_loss": 4.679632078128385, + "tokens_seen": 115343360 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048739217652958876, + "loss": 3.2653, + "theoretical_loss": 4.679270955873918, + "tokens_seen": 115408896 + }, + { + "epoch": 0.03, + "learning_rate": 0.00048738214643931794, + "loss": 3.4833, + "theoretical_loss": 4.678910096009329, + "tokens_seen": 115474432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048737211634904717, + "loss": 3.3117, + "theoretical_loss": 4.678549498195213, + "tokens_seen": 115539968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873620862587763, + "loss": 3.3255, + "theoretical_loss": 4.678189162092798, + "tokens_seen": 115605504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048735205616850553, + "loss": 3.2552, + "theoretical_loss": 4.677829087363937, + "tokens_seen": 115671040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873420260782347, + "loss": 3.2912, + "theoretical_loss": 4.677469273671118, + "tokens_seen": 115736576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873319959879639, + "loss": 3.1695, + "theoretical_loss": 4.677109720677448, + "tokens_seen": 115802112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004873219658976931, + "loss": 3.2226, + "theoretical_loss": 4.676750428046665, + "tokens_seen": 115867648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048731193580742226, + "loss": 3.2803, + "theoretical_loss": 4.676391395443128, + "tokens_seen": 115933184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048730190571715144, + "loss": 3.5568, + "theoretical_loss": 4.6760326225318165, + "tokens_seen": 115998720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872918756268807, + "loss": 3.339, + "theoretical_loss": 4.675674108978335, + "tokens_seen": 116064256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872818455366098, + "loss": 3.1442, + "theoretical_loss": 4.675315854448903, + "tokens_seen": 116129792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048727181544633904, + "loss": 3.1936, + "theoretical_loss": 4.674957858610361, + "tokens_seen": 116195328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048726178535606816, + "loss": 3.2333, + "theoretical_loss": 4.674600121130163, + "tokens_seen": 116260864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 74658, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9736719131469727, + "objective/train/theoretical_loss": 4.674242641676377, + "objective/train/tokens_used": 136786400, + "theoretical_loss": 4.674242641676377, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872517552657974, + "loss": 3.3693, + "theoretical_loss": 4.674242641676377, + "tokens_seen": 116326400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872417251755266, + "loss": 3.3527, + "theoretical_loss": 4.673885419917689, + "tokens_seen": 116391936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048723169508525576, + "loss": 3.5978, + "theoretical_loss": 4.673528455523392, + "tokens_seen": 116457472 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487221664994985, + "loss": 3.416, + "theoretical_loss": 4.673171748163389, + "tokens_seen": 116523008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004872116349047141, + "loss": 3.3723, + "theoretical_loss": 4.672815297508194, + "tokens_seen": 116588544 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048720160481444336, + "loss": 3.1412, + "theoretical_loss": 4.672459103228929, + "tokens_seen": 116654080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048719157472417254, + "loss": 3.6399, + "theoretical_loss": 4.672103164997319, + "tokens_seen": 116719616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871815446339017, + "loss": 3.3651, + "theoretical_loss": 4.671747482485696, + "tokens_seen": 116785152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871715145436309, + "loss": 3.3461, + "theoretical_loss": 4.671392055366992, + "tokens_seen": 116850688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048716148445336014, + "loss": 3.2057, + "theoretical_loss": 4.671036883314743, + "tokens_seen": 116916224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048715145436308927, + "loss": 3.1564, + "theoretical_loss": 4.670681966003084, + "tokens_seen": 116981760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871414242728185, + "loss": 3.3224, + "theoretical_loss": 4.670327303106748, + "tokens_seen": 117047296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048713139418254763, + "loss": 3.5285, + "theoretical_loss": 4.6699728943010665, + "tokens_seen": 117112832 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048712136409227686, + "loss": 3.2677, + "theoretical_loss": 4.669618739261967, + "tokens_seen": 117178368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048711133400200604, + "loss": 3.1055, + "theoretical_loss": 4.669264837665969, + "tokens_seen": 117243904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004871013039117352, + "loss": 3.3293, + "theoretical_loss": 4.668911189190185, + "tokens_seen": 117309440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870912738214644, + "loss": 3.3489, + "theoretical_loss": 4.668557793512322, + "tokens_seen": 117374976 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870812437311936, + "loss": 3.1125, + "theoretical_loss": 4.668204650310676, + "tokens_seen": 117440512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048707121364092277, + "loss": 3.2515, + "theoretical_loss": 4.667851759264128, + "tokens_seen": 117506048 + }, + { + "epoch": 0.04, + "learning_rate": 0.000487061183550652, + "loss": 3.3586, + "theoretical_loss": 4.667499120052151, + "tokens_seen": 117571584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048705115346038113, + "loss": 3.2803, + "theoretical_loss": 4.6671467323548, + "tokens_seen": 117637120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048704112337011037, + "loss": 3.3058, + "theoretical_loss": 4.666794595852718, + "tokens_seen": 117702656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048703109327983955, + "loss": 3.2752, + "theoretical_loss": 4.666442710227129, + "tokens_seen": 117768192 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048702106318956873, + "loss": 3.3295, + "theoretical_loss": 4.666091075159839, + "tokens_seen": 117833728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870110330992979, + "loss": 3.5682, + "theoretical_loss": 4.665739690333233, + "tokens_seen": 117899264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 75891, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9445178508758545, + "objective/train/theoretical_loss": 4.665388555430277, + "objective/train/tokens_used": 138424800, + "theoretical_loss": 4.665388555430277, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004870010030090271, + "loss": 3.4408, + "theoretical_loss": 4.665388555430277, + "tokens_seen": 117964800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048699097291875627, + "loss": 3.42, + "theoretical_loss": 4.665037670134513, + "tokens_seen": 118030336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869809428284855, + "loss": 3.4118, + "theoretical_loss": 4.66468703413006, + "tokens_seen": 118095872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048697091273821463, + "loss": 2.8531, + "theoretical_loss": 4.664336647101612, + "tokens_seen": 118161408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048696088264794387, + "loss": 3.5486, + "theoretical_loss": 4.663986508734435, + "tokens_seen": 118226944 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486950852557673, + "loss": 3.3137, + "theoretical_loss": 4.663636618714368, + "tokens_seen": 118292480 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048694082246740223, + "loss": 3.2493, + "theoretical_loss": 4.663286976727821, + "tokens_seen": 118358016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869307923771314, + "loss": 3.4941, + "theoretical_loss": 4.6629375824617725, + "tokens_seen": 118423552 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869207622868606, + "loss": 3.1022, + "theoretical_loss": 4.6625884356037695, + "tokens_seen": 118489088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004869107321965898, + "loss": 3.2497, + "theoretical_loss": 4.662239535841925, + "tokens_seen": 118554624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048690070210631896, + "loss": 3.465, + "theoretical_loss": 4.661890882864919, + "tokens_seen": 118620160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048689067201604814, + "loss": 3.3506, + "theoretical_loss": 4.661542476361992, + "tokens_seen": 118685696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048688064192577737, + "loss": 3.2192, + "theoretical_loss": 4.661194316022952, + "tokens_seen": 118751232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868706118355065, + "loss": 3.274, + "theoretical_loss": 4.660846401538165, + "tokens_seen": 118816768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048686058174523573, + "loss": 3.1325, + "theoretical_loss": 4.6604987325985565, + "tokens_seen": 118882304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868505516549649, + "loss": 3.241, + "theoretical_loss": 4.660151308895614, + "tokens_seen": 118947840 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868405215646941, + "loss": 3.457, + "theoretical_loss": 4.659804130121378, + "tokens_seen": 119013376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868304914744233, + "loss": 3.2732, + "theoretical_loss": 4.659457195968448, + "tokens_seen": 119078912 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048682046138415246, + "loss": 3.1738, + "theoretical_loss": 4.659110506129977, + "tokens_seen": 119144448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048681043129388164, + "loss": 3.2882, + "theoretical_loss": 4.6587640602996725, + "tokens_seen": 119209984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004868004012036109, + "loss": 3.4992, + "theoretical_loss": 4.658417858171794, + "tokens_seen": 119275520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048679037111334, + "loss": 3.2217, + "theoretical_loss": 4.6580718994411505, + "tokens_seen": 119341056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048678034102306924, + "loss": 3.3294, + "theoretical_loss": 4.657726183803101, + "tokens_seen": 119406592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048677031093279836, + "loss": 3.0235, + "theoretical_loss": 4.657380710953554, + "tokens_seen": 119472128 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867602808425276, + "loss": 3.2772, + "theoretical_loss": 4.657035480588962, + "tokens_seen": 119537664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 76546, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.836437225341797, + "objective/train/theoretical_loss": 4.656690492406327, + "objective/train/tokens_used": 140063200, + "theoretical_loss": 4.656690492406327, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867502507522568, + "loss": 3.395, + "theoretical_loss": 4.656690492406327, + "tokens_seen": 119603200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048674022066198596, + "loss": 3.2332, + "theoretical_loss": 4.6563457461031925, + "tokens_seen": 119668736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048673019057171514, + "loss": 3.3067, + "theoretical_loss": 4.656001241377647, + "tokens_seen": 119734272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867201604814443, + "loss": 3.4375, + "theoretical_loss": 4.655656977928318, + "tokens_seen": 119799808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004867101303911735, + "loss": 3.3728, + "theoretical_loss": 4.655312955454378, + "tokens_seen": 119865344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048670010030090274, + "loss": 3.4504, + "theoretical_loss": 4.654969173655536, + "tokens_seen": 119930880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048669007021063187, + "loss": 3.168, + "theoretical_loss": 4.654625632232037, + "tokens_seen": 119996416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866800401203611, + "loss": 3.1419, + "theoretical_loss": 4.654282330884667, + "tokens_seen": 120061952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004866700100300903, + "loss": 3.483, + "theoretical_loss": 4.653939269314744, + "tokens_seen": 120127488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048665997993981947, + "loss": 3.1995, + "theoretical_loss": 4.653596447224125, + "tokens_seen": 120193024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048664994984954865, + "loss": 3.0174, + "theoretical_loss": 4.653253864315193, + "tokens_seen": 120258560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048663991975927783, + "loss": 3.2561, + "theoretical_loss": 4.6529115202908695, + "tokens_seen": 120324096 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486629889669007, + "loss": 3.3328, + "theoretical_loss": 4.652569414854604, + "tokens_seen": 120389632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048661985957873624, + "loss": 3.2924, + "theoretical_loss": 4.652227547710373, + "tokens_seen": 120455168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048660982948846537, + "loss": 3.2271, + "theoretical_loss": 4.651885918562684, + "tokens_seen": 120520704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865997993981946, + "loss": 3.2564, + "theoretical_loss": 4.65154452711657, + "tokens_seen": 120586240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048658976930792373, + "loss": 3.2625, + "theoretical_loss": 4.651203373077593, + "tokens_seen": 120651776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048657973921765297, + "loss": 3.1227, + "theoretical_loss": 4.650862456151833, + "tokens_seen": 120717312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048656970912738215, + "loss": 3.2546, + "theoretical_loss": 4.650521776045898, + "tokens_seen": 120782848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048655967903711133, + "loss": 3.2324, + "theoretical_loss": 4.650181332466918, + "tokens_seen": 120848384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865496489468405, + "loss": 3.3792, + "theoretical_loss": 4.64984112512254, + "tokens_seen": 120913920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048653961885656975, + "loss": 3.4118, + "theoretical_loss": 4.6495011537209345, + "tokens_seen": 120979456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865295887662989, + "loss": 3.3581, + "theoretical_loss": 4.649161417970789, + "tokens_seen": 121044992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004865195586760281, + "loss": 3.4152, + "theoretical_loss": 4.648821917581308, + "tokens_seen": 121110528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048650952858575724, + "loss": 3.4244, + "theoretical_loss": 4.648482652262211, + "tokens_seen": 121176064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 77942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9632604122161865, + "objective/train/theoretical_loss": 4.648143621723734, + "objective/train/tokens_used": 141701600, + "theoretical_loss": 4.648143621723734, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048649949849548647, + "loss": 3.2822, + "theoretical_loss": 4.648143621723734, + "tokens_seen": 121241600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048648946840521565, + "loss": 3.0832, + "theoretical_loss": 4.6478048256766265, + "tokens_seen": 121307136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048647943831494483, + "loss": 3.3079, + "theoretical_loss": 4.647466263832148, + "tokens_seen": 121372672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048646940822467407, + "loss": 3.2723, + "theoretical_loss": 4.647127935902072, + "tokens_seen": 121438208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864593781344032, + "loss": 3.5748, + "theoretical_loss": 4.646789841598681, + "tokens_seen": 121503744 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048644934804413243, + "loss": 3.1642, + "theoretical_loss": 4.646451980634765, + "tokens_seen": 121569280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864393179538616, + "loss": 3.1145, + "theoretical_loss": 4.646114352723625, + "tokens_seen": 121634816 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004864292878635908, + "loss": 3.5071, + "theoretical_loss": 4.645776957579065, + "tokens_seen": 121700352 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048641925777332, + "loss": 3.4015, + "theoretical_loss": 4.645439794915395, + "tokens_seen": 121765888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048640922768304916, + "loss": 3.2914, + "theoretical_loss": 4.6451028644474315, + "tokens_seen": 121831424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048639919759277834, + "loss": 3.2374, + "theoretical_loss": 4.644766165890491, + "tokens_seen": 121896960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048638916750250757, + "loss": 3.2349, + "theoretical_loss": 4.644429698960394, + "tokens_seen": 121962496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863791374122367, + "loss": 3.4859, + "theoretical_loss": 4.644093463373462, + "tokens_seen": 122028032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048636910732196593, + "loss": 3.2108, + "theoretical_loss": 4.643757458846514, + "tokens_seen": 122093568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863590772316951, + "loss": 3.2694, + "theoretical_loss": 4.643421685096866, + "tokens_seen": 122159104 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863490471414243, + "loss": 3.525, + "theoretical_loss": 4.643086141842337, + "tokens_seen": 122224640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863390170511535, + "loss": 3.4927, + "theoretical_loss": 4.642750828801238, + "tokens_seen": 122290176 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048632898696088266, + "loss": 3.4461, + "theoretical_loss": 4.642415745692375, + "tokens_seen": 122355712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048631895687061184, + "loss": 3.3788, + "theoretical_loss": 4.6420808922350485, + "tokens_seen": 122421248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004863089267803411, + "loss": 3.3053, + "theoretical_loss": 4.641746268149052, + "tokens_seen": 122486784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862988966900702, + "loss": 3.2416, + "theoretical_loss": 4.64141187315467, + "tokens_seen": 122552320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048628886659979944, + "loss": 3.0169, + "theoretical_loss": 4.641077706972679, + "tokens_seen": 122617856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048627883650952857, + "loss": 3.3138, + "theoretical_loss": 4.6407437693243425, + "tokens_seen": 122683392 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862688064192578, + "loss": 3.1577, + "theoretical_loss": 4.640410059931414, + "tokens_seen": 122748928 + }, + { + "epoch": 0.04, + "learning_rate": 0.000486258776328987, + "loss": 3.3863, + "theoretical_loss": 4.640076578516132, + "tokens_seen": 122814464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 78541, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7341437339782715, + "objective/train/theoretical_loss": 4.639743324801225, + "objective/train/tokens_used": 143340000, + "theoretical_loss": 4.639743324801225, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048624874623871616, + "loss": 3.1381, + "theoretical_loss": 4.639743324801225, + "tokens_seen": 122880000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048623871614844534, + "loss": 3.3316, + "theoretical_loss": 4.639410298509901, + "tokens_seen": 122945536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862286860581745, + "loss": 3.2356, + "theoretical_loss": 4.639077499365856, + "tokens_seen": 123011072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004862186559679037, + "loss": 3.355, + "theoretical_loss": 4.638744927093268, + "tokens_seen": 123076608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048620862587763294, + "loss": 3.3971, + "theoretical_loss": 4.6384125814167945, + "tokens_seen": 123142144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048619859578736207, + "loss": 3.0734, + "theoretical_loss": 4.638080462061575, + "tokens_seen": 123207680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861885656970913, + "loss": 3.1079, + "theoretical_loss": 4.63774856875323, + "tokens_seen": 123273216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861785356068205, + "loss": 3.3234, + "theoretical_loss": 4.637416901217852, + "tokens_seen": 123338752 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048616850551654967, + "loss": 3.3152, + "theoretical_loss": 4.63708545918202, + "tokens_seen": 123404288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048615847542627885, + "loss": 3.345, + "theoretical_loss": 4.636754242372782, + "tokens_seen": 123469824 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048614844533600803, + "loss": 3.3265, + "theoretical_loss": 4.636423250517664, + "tokens_seen": 123535360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861384152457372, + "loss": 3.3053, + "theoretical_loss": 4.636092483344665, + "tokens_seen": 123600896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048612838515546644, + "loss": 3.3878, + "theoretical_loss": 4.635761940582256, + "tokens_seen": 123666432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048611835506519557, + "loss": 3.3957, + "theoretical_loss": 4.635431621959384, + "tokens_seen": 123731968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004861083249749248, + "loss": 3.4768, + "theoretical_loss": 4.63510152720546, + "tokens_seen": 123797504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048609829488465393, + "loss": 3.2736, + "theoretical_loss": 4.634771656050372, + "tokens_seen": 123863040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048608826479438317, + "loss": 3.1239, + "theoretical_loss": 4.63444200822447, + "tokens_seen": 123928576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048607823470411235, + "loss": 3.2963, + "theoretical_loss": 4.6341125834585775, + "tokens_seen": 123994112 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048606820461384153, + "loss": 3.4747, + "theoretical_loss": 4.633783381483978, + "tokens_seen": 124059648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860581745235707, + "loss": 3.374, + "theoretical_loss": 4.633454402032427, + "tokens_seen": 124125184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048604814443329995, + "loss": 3.4227, + "theoretical_loss": 4.63312564483614, + "tokens_seen": 124190720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860381143430291, + "loss": 3.6141, + "theoretical_loss": 4.632797109627798, + "tokens_seen": 124256256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004860280842527583, + "loss": 3.2804, + "theoretical_loss": 4.632468796140543, + "tokens_seen": 124321792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048601805416248744, + "loss": 3.4935, + "theoretical_loss": 4.632140704107979, + "tokens_seen": 124387328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048600802407221667, + "loss": 3.3112, + "theoretical_loss": 4.631812833264169, + "tokens_seen": 124452864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 79709, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.385240316390991, + "objective/train/theoretical_loss": 4.631485183343639, + "objective/train/tokens_used": 144978400, + "theoretical_loss": 4.631485183343639, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048599799398194585, + "loss": 3.2349, + "theoretical_loss": 4.631485183343639, + "tokens_seen": 124518400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048598796389167503, + "loss": 3.3546, + "theoretical_loss": 4.631157754081366, + "tokens_seen": 124583936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859779338014042, + "loss": 3.088, + "theoretical_loss": 4.630830545212792, + "tokens_seen": 124649472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859679037111334, + "loss": 3.4382, + "theoretical_loss": 4.63050355647381, + "tokens_seen": 124715008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859578736208626, + "loss": 3.3753, + "theoretical_loss": 4.630176787600771, + "tokens_seen": 124780544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859478435305918, + "loss": 3.1749, + "theoretical_loss": 4.6298502383304765, + "tokens_seen": 124846080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048593781344032094, + "loss": 3.1883, + "theoretical_loss": 4.629523908400184, + "tokens_seen": 124911616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859277833500502, + "loss": 3.4012, + "theoretical_loss": 4.6291977975476035, + "tokens_seen": 124977152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004859177532597793, + "loss": 3.1556, + "theoretical_loss": 4.628871905510893, + "tokens_seen": 125042688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048590772316950854, + "loss": 3.5815, + "theoretical_loss": 4.628546232028661, + "tokens_seen": 125108224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858976930792377, + "loss": 3.2682, + "theoretical_loss": 4.62822077683997, + "tokens_seen": 125173760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858876629889669, + "loss": 3.685, + "theoretical_loss": 4.627895539684323, + "tokens_seen": 125239296 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858776328986961, + "loss": 3.2897, + "theoretical_loss": 4.627570520301673, + "tokens_seen": 125304832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858676028084253, + "loss": 3.1734, + "theoretical_loss": 4.627245718432423, + "tokens_seen": 125370368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048585757271815444, + "loss": 3.0723, + "theoretical_loss": 4.626921133817413, + "tokens_seen": 125435904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858475426278837, + "loss": 3.1019, + "theoretical_loss": 4.626596766197936, + "tokens_seen": 125501440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858375125376128, + "loss": 3.21, + "theoretical_loss": 4.62627261531572, + "tokens_seen": 125566976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048582748244734204, + "loss": 3.1586, + "theoretical_loss": 4.625948680912939, + "tokens_seen": 125632512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858174523570712, + "loss": 3.1464, + "theoretical_loss": 4.625624962732207, + "tokens_seen": 125698048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004858074222668004, + "loss": 3.5956, + "theoretical_loss": 4.62530146051658, + "tokens_seen": 125763584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857973921765296, + "loss": 3.2982, + "theoretical_loss": 4.624978174009549, + "tokens_seen": 125829120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048578736208625877, + "loss": 2.8431, + "theoretical_loss": 4.624655102955048, + "tokens_seen": 125894656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048577733199598795, + "loss": 3.2039, + "theoretical_loss": 4.624332247097442, + "tokens_seen": 125960192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857673019057172, + "loss": 3.255, + "theoretical_loss": 4.624009606181539, + "tokens_seen": 126025728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857572718154463, + "loss": 3.3476, + "theoretical_loss": 4.623687179952577, + "tokens_seen": 126091264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 80984, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0424399375915527, + "objective/train/theoretical_loss": 4.623364968156229, + "objective/train/tokens_used": 146616800, + "theoretical_loss": 4.623364968156229, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048574724172517554, + "loss": 3.1198, + "theoretical_loss": 4.623364968156229, + "tokens_seen": 126156800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048573721163490467, + "loss": 3.2627, + "theoretical_loss": 4.623042970538604, + "tokens_seen": 126222336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004857271815446339, + "loss": 3.2619, + "theoretical_loss": 4.622721186846241, + "tokens_seen": 126287872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048571715145436314, + "loss": 3.077, + "theoretical_loss": 4.62239961682611, + "tokens_seen": 126353408 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048570712136409227, + "loss": 3.4283, + "theoretical_loss": 4.622078260225611, + "tokens_seen": 126418944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856970912738215, + "loss": 3.1504, + "theoretical_loss": 4.621757116792576, + "tokens_seen": 126484480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856870611835507, + "loss": 3.2759, + "theoretical_loss": 4.621436186275262, + "tokens_seen": 126550016 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048567703109327987, + "loss": 3.2942, + "theoretical_loss": 4.6211154684223565, + "tokens_seen": 126615552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048566700100300905, + "loss": 3.462, + "theoretical_loss": 4.6207949629829725, + "tokens_seen": 126681088 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048565697091273823, + "loss": 2.9686, + "theoretical_loss": 4.620474669706647, + "tokens_seen": 126746624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004856469408224674, + "loss": 3.3596, + "theoretical_loss": 4.620154588343343, + "tokens_seen": 126812160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048563691073219664, + "loss": 3.3793, + "theoretical_loss": 4.619834718643448, + "tokens_seen": 126877696 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048562688064192577, + "loss": 3.327, + "theoretical_loss": 4.619515060357771, + "tokens_seen": 126943232 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485616850551655, + "loss": 3.5938, + "theoretical_loss": 4.619195613237542, + "tokens_seen": 127008768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048560682046138413, + "loss": 3.1359, + "theoretical_loss": 4.6188763770344154, + "tokens_seen": 127074304 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048559679037111337, + "loss": 3.6939, + "theoretical_loss": 4.618557351500461, + "tokens_seen": 127139840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048558676028084255, + "loss": 3.3604, + "theoretical_loss": 4.618238536388171, + "tokens_seen": 127205376 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048557673019057173, + "loss": 3.4903, + "theoretical_loss": 4.617919931450454, + "tokens_seen": 127270912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855667001003009, + "loss": 3.3289, + "theoretical_loss": 4.617601536440637, + "tokens_seen": 127336448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048555667001003015, + "loss": 3.1247, + "theoretical_loss": 4.617283351112462, + "tokens_seen": 127401984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855466399197593, + "loss": 3.5764, + "theoretical_loss": 4.616965375220088, + "tokens_seen": 127467520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004855366098294885, + "loss": 3.419, + "theoretical_loss": 4.616647608518086, + "tokens_seen": 127533056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048552657973921764, + "loss": 3.2447, + "theoretical_loss": 4.616330050761444, + "tokens_seen": 127598592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048551654964894687, + "loss": 3.0136, + "theoretical_loss": 4.616012701705559, + "tokens_seen": 127664128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048550651955867605, + "loss": 3.1339, + "theoretical_loss": 4.615695561106241, + "tokens_seen": 127729664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 81703, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6864616870880127, + "objective/train/theoretical_loss": 4.6153786287197125, + "objective/train/tokens_used": 148255200, + "theoretical_loss": 4.6153786287197125, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048549648946840523, + "loss": 3.2948, + "theoretical_loss": 4.6153786287197125, + "tokens_seen": 127795200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854864593781344, + "loss": 3.0792, + "theoretical_loss": 4.615061904302603, + "tokens_seen": 127860736 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854764292878636, + "loss": 3.1573, + "theoretical_loss": 4.614745387611955, + "tokens_seen": 127926272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854663991975928, + "loss": 3.4941, + "theoretical_loss": 4.614429078405214, + "tokens_seen": 127991808 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485456369107322, + "loss": 3.2639, + "theoretical_loss": 4.6141129764402375, + "tokens_seen": 128057344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048544633901705114, + "loss": 3.2058, + "theoretical_loss": 4.613797081475287, + "tokens_seen": 128122880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854363089267804, + "loss": 3.4029, + "theoretical_loss": 4.613481393269028, + "tokens_seen": 128188416 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854262788365095, + "loss": 3.5993, + "theoretical_loss": 4.613165911580533, + "tokens_seen": 128253952 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048541624874623874, + "loss": 3.1374, + "theoretical_loss": 4.612850636169279, + "tokens_seen": 128319488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004854062186559679, + "loss": 3.3721, + "theoretical_loss": 4.612535566795142, + "tokens_seen": 128385024 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853961885656971, + "loss": 3.1656, + "theoretical_loss": 4.612220703218403, + "tokens_seen": 128450560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853861584754263, + "loss": 3.4822, + "theoretical_loss": 4.611906045199744, + "tokens_seen": 128516096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853761283851555, + "loss": 3.1366, + "theoretical_loss": 4.611591592500245, + "tokens_seen": 128581632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048536609829488464, + "loss": 3.3858, + "theoretical_loss": 4.611277344881389, + "tokens_seen": 128647168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853560682046139, + "loss": 3.1729, + "theoretical_loss": 4.610963302105051, + "tokens_seen": 128712704 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485346038114343, + "loss": 3.2684, + "theoretical_loss": 4.610649463933511, + "tokens_seen": 128778240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048533600802407224, + "loss": 3.1187, + "theoretical_loss": 4.61033583012944, + "tokens_seen": 128843776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853259779338014, + "loss": 3.2702, + "theoretical_loss": 4.610022400455909, + "tokens_seen": 128909312 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853159478435306, + "loss": 3.1401, + "theoretical_loss": 4.6097091746763805, + "tokens_seen": 128974848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004853059177532598, + "loss": 3.0625, + "theoretical_loss": 4.609396152554714, + "tokens_seen": 129040384 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048529588766298897, + "loss": 3.4469, + "theoretical_loss": 4.60908333385516, + "tokens_seen": 129105920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048528585757271815, + "loss": 3.2416, + "theoretical_loss": 4.608770718342361, + "tokens_seen": 129171456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852758274824474, + "loss": 3.2241, + "theoretical_loss": 4.6084583057813555, + "tokens_seen": 129236992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852657973921765, + "loss": 3.2384, + "theoretical_loss": 4.608146095937567, + "tokens_seen": 129302528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048525576730190574, + "loss": 3.4009, + "theoretical_loss": 4.607834088576811, + "tokens_seen": 129368064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 82831, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1748886108398438, + "objective/train/theoretical_loss": 4.607522283465294, + "objective/train/tokens_used": 149893600, + "theoretical_loss": 4.607522283465294, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048524573721163487, + "loss": 3.2513, + "theoretical_loss": 4.607522283465294, + "tokens_seen": 129433600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852357071213641, + "loss": 3.3024, + "theoretical_loss": 4.607210680369608, + "tokens_seen": 129499136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004852256770310933, + "loss": 3.5175, + "theoretical_loss": 4.606899279056734, + "tokens_seen": 129564672 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048521564694082247, + "loss": 3.1854, + "theoretical_loss": 4.606588079294038, + "tokens_seen": 129630208 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048520561685055165, + "loss": 3.1448, + "theoretical_loss": 4.606277080849273, + "tokens_seen": 129695744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851955867602809, + "loss": 3.2784, + "theoretical_loss": 4.605966283490575, + "tokens_seen": 129761280 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048518555667001, + "loss": 3.055, + "theoretical_loss": 4.6056556869864655, + "tokens_seen": 129826816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048517552657973925, + "loss": 3.4625, + "theoretical_loss": 4.60534529110585, + "tokens_seen": 129892352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851654964894684, + "loss": 3.2055, + "theoretical_loss": 4.605035095618014, + "tokens_seen": 129957888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851554663991976, + "loss": 3.3952, + "theoretical_loss": 4.604725100292626, + "tokens_seen": 130023424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851454363089268, + "loss": 3.193, + "theoretical_loss": 4.604415304899735, + "tokens_seen": 130088960 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048513540621865597, + "loss": 3.3349, + "theoretical_loss": 4.604105709209769, + "tokens_seen": 130154496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048512537612838515, + "loss": 3.4078, + "theoretical_loss": 4.603796312993536, + "tokens_seen": 130220032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048511534603811433, + "loss": 3.3374, + "theoretical_loss": 4.603487116022222, + "tokens_seen": 130285568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004851053159478435, + "loss": 3.2205, + "theoretical_loss": 4.60317811806739, + "tokens_seen": 130351104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048509528585757275, + "loss": 3.3053, + "theoretical_loss": 4.60286931890098, + "tokens_seen": 130416640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850852557673019, + "loss": 3.3346, + "theoretical_loss": 4.602560718295308, + "tokens_seen": 130482176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850752256770311, + "loss": 3.3864, + "theoretical_loss": 4.602252316023064, + "tokens_seen": 130547712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048506519558676024, + "loss": 3.4327, + "theoretical_loss": 4.601944111857312, + "tokens_seen": 130613248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850551654964895, + "loss": 3.2583, + "theoretical_loss": 4.601636105571492, + "tokens_seen": 130678784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048504513540621866, + "loss": 3.4605, + "theoretical_loss": 4.601328296939414, + "tokens_seen": 130744320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048503510531594784, + "loss": 3.2544, + "theoretical_loss": 4.601020685735259, + "tokens_seen": 130809856 + }, + { + "epoch": 0.04, + "learning_rate": 0.000485025075225677, + "loss": 3.2836, + "theoretical_loss": 4.600713271733582, + "tokens_seen": 130875392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048501504513540625, + "loss": 3.4586, + "theoretical_loss": 4.600406054709307, + "tokens_seen": 130940928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004850050150451354, + "loss": 3.3115, + "theoretical_loss": 4.600099034437726, + "tokens_seen": 131006464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 83204, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2750132083892822, + "objective/train/theoretical_loss": 4.5997922106945, + "objective/train/tokens_used": 151532000, + "theoretical_loss": 4.5997922106945, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849949849548646, + "loss": 3.3318, + "theoretical_loss": 4.5997922106945, + "tokens_seen": 131072000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849849548645938, + "loss": 3.5144, + "theoretical_loss": 4.5994855832556585, + "tokens_seen": 131137536 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484974924774323, + "loss": 3.4244, + "theoretical_loss": 4.5991791518976, + "tokens_seen": 131203072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849648946840522, + "loss": 3.2389, + "theoretical_loss": 4.5988729163970845, + "tokens_seen": 131268608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048495486459378134, + "loss": 3.3639, + "theoretical_loss": 4.59856687653124, + "tokens_seen": 131334144 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849448345035106, + "loss": 3.2156, + "theoretical_loss": 4.598261032077561, + "tokens_seen": 131399680 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849348044132397, + "loss": 3.5008, + "theoretical_loss": 4.597955382813903, + "tokens_seen": 131465216 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048492477432296894, + "loss": 3.4594, + "theoretical_loss": 4.597649928518486, + "tokens_seen": 131530752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849147442326981, + "loss": 3.3812, + "theoretical_loss": 4.597344668969892, + "tokens_seen": 131596288 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004849047141424273, + "loss": 3.3369, + "theoretical_loss": 4.597039603947064, + "tokens_seen": 131661824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848946840521565, + "loss": 3.2768, + "theoretical_loss": 4.596734733229306, + "tokens_seen": 131727360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848846539618857, + "loss": 2.9459, + "theoretical_loss": 4.596430056596285, + "tokens_seen": 131792896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048487462387161484, + "loss": 3.0683, + "theoretical_loss": 4.596125573828021, + "tokens_seen": 131858432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848645937813441, + "loss": 3.3636, + "theoretical_loss": 4.595821284704899, + "tokens_seen": 131923968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848545636910732, + "loss": 3.2567, + "theoretical_loss": 4.5955171890076585, + "tokens_seen": 131989504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048484453360080244, + "loss": 3.5107, + "theoretical_loss": 4.595213286517394, + "tokens_seen": 132055040 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848345035105316, + "loss": 3.3222, + "theoretical_loss": 4.594909577015562, + "tokens_seen": 132120576 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004848244734202608, + "loss": 3.4088, + "theoretical_loss": 4.594606060283969, + "tokens_seen": 132186112 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048481444332999, + "loss": 3.3058, + "theoretical_loss": 4.594302736104778, + "tokens_seen": 132251648 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048480441323971917, + "loss": 3.3761, + "theoretical_loss": 4.593999604260507, + "tokens_seen": 132317184 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048479438314944835, + "loss": 3.115, + "theoretical_loss": 4.593696664534026, + "tokens_seen": 132382720 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847843530591776, + "loss": 3.0734, + "theoretical_loss": 4.593393916708557, + "tokens_seen": 132448256 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847743229689067, + "loss": 3.4007, + "theoretical_loss": 4.593091360567676, + "tokens_seen": 132513792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048476429287863594, + "loss": 3.4221, + "theoretical_loss": 4.59278899589531, + "tokens_seen": 132579328 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048475426278836507, + "loss": 3.3316, + "theoretical_loss": 4.592486822475731, + "tokens_seen": 132644864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 84183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4309723377227783, + "objective/train/theoretical_loss": 4.592184840093566, + "objective/train/tokens_used": 153170400, + "theoretical_loss": 4.592184840093566, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847442326980943, + "loss": 3.1609, + "theoretical_loss": 4.592184840093566, + "tokens_seen": 132710400 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847342026078235, + "loss": 3.3487, + "theoretical_loss": 4.59188304853379, + "tokens_seen": 132775936 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048472417251755267, + "loss": 3.2624, + "theoretical_loss": 4.591581447581723, + "tokens_seen": 132841472 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048471414242728185, + "loss": 3.0568, + "theoretical_loss": 4.591280037023036, + "tokens_seen": 132907008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004847041123370111, + "loss": 3.3944, + "theoretical_loss": 4.590978816643744, + "tokens_seen": 132972544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846940822467402, + "loss": 3.375, + "theoretical_loss": 4.590677786230208, + "tokens_seen": 133038080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048468405215646945, + "loss": 3.1915, + "theoretical_loss": 4.590376945569136, + "tokens_seen": 133103616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846740220661986, + "loss": 3.2734, + "theoretical_loss": 4.590076294447578, + "tokens_seen": 133169152 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846639919759278, + "loss": 3.1604, + "theoretical_loss": 4.589775832652929, + "tokens_seen": 133234688 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484653961885657, + "loss": 3.1837, + "theoretical_loss": 4.589475559972927, + "tokens_seen": 133300224 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048464393179538617, + "loss": 3.3031, + "theoretical_loss": 4.589175476195651, + "tokens_seen": 133365760 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048463390170511535, + "loss": 3.1795, + "theoretical_loss": 4.588875581109523, + "tokens_seen": 133431296 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048462387161484453, + "loss": 3.2039, + "theoretical_loss": 4.588575874503304, + "tokens_seen": 133496832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004846138415245737, + "loss": 3.4852, + "theoretical_loss": 4.588276356166096, + "tokens_seen": 133562368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048460381143430295, + "loss": 3.3087, + "theoretical_loss": 4.587977025887342, + "tokens_seen": 133627904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845937813440321, + "loss": 3.1137, + "theoretical_loss": 4.58767788345682, + "tokens_seen": 133693440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845837512537613, + "loss": 3.2587, + "theoretical_loss": 4.5873789286646485, + "tokens_seen": 133758976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048457372116349044, + "loss": 3.3583, + "theoretical_loss": 4.587080161301283, + "tokens_seen": 133824512 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845636910732197, + "loss": 3.2645, + "theoretical_loss": 4.586781581157513, + "tokens_seen": 133890048 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048455366098294886, + "loss": 3.289, + "theoretical_loss": 4.58648318802447, + "tokens_seen": 133955584 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048454363089267804, + "loss": 2.9687, + "theoretical_loss": 4.586184981693611, + "tokens_seen": 134021120 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845336008024072, + "loss": 3.1314, + "theoretical_loss": 4.585886961956737, + "tokens_seen": 134086656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048452357071213645, + "loss": 3.6532, + "theoretical_loss": 4.585589128605976, + "tokens_seen": 134152192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845135406218656, + "loss": 3.1767, + "theoretical_loss": 4.585291481433792, + "tokens_seen": 134217728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004845035105315948, + "loss": 3.1699, + "theoretical_loss": 4.584994020232981, + "tokens_seen": 134283264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 84861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.49385142326355, + "objective/train/theoretical_loss": 4.58469674479667, + "objective/train/tokens_used": 154808800, + "theoretical_loss": 4.58469674479667, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048449348044132394, + "loss": 3.3437, + "theoretical_loss": 4.58469674479667, + "tokens_seen": 134348800 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844834503510532, + "loss": 3.3897, + "theoretical_loss": 4.584399654918316, + "tokens_seen": 134414336 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048447342026078236, + "loss": 3.2379, + "theoretical_loss": 4.584102750391709, + "tokens_seen": 134479872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048446339017051154, + "loss": 3.4963, + "theoretical_loss": 4.583806031010965, + "tokens_seen": 134545408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844533600802407, + "loss": 3.4515, + "theoretical_loss": 4.5835094965705325, + "tokens_seen": 134610944 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844433299899699, + "loss": 3.323, + "theoretical_loss": 4.5832131468651855, + "tokens_seen": 134676480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844332998996991, + "loss": 3.0751, + "theoretical_loss": 4.582916981690026, + "tokens_seen": 134742016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844232698094283, + "loss": 3.2973, + "theoretical_loss": 4.582621000840484, + "tokens_seen": 134807552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048441323971915745, + "loss": 3.3443, + "theoretical_loss": 4.582325204112314, + "tokens_seen": 134873088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004844032096288867, + "loss": 3.2763, + "theoretical_loss": 4.582029591301595, + "tokens_seen": 134938624 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048439317953861586, + "loss": 3.3043, + "theoretical_loss": 4.581734162204734, + "tokens_seen": 135004160 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048438314944834504, + "loss": 3.2092, + "theoretical_loss": 4.5814389166184615, + "tokens_seen": 135069696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843731193580742, + "loss": 3.2379, + "theoretical_loss": 4.581143854339828, + "tokens_seen": 135135232 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843630892678034, + "loss": 3.2659, + "theoretical_loss": 4.580848975166209, + "tokens_seen": 135200768 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843530591775326, + "loss": 3.0079, + "theoretical_loss": 4.580554278895304, + "tokens_seen": 135266304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843430290872618, + "loss": 3.2396, + "theoretical_loss": 4.58025976532513, + "tokens_seen": 135331840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048433299899699095, + "loss": 3.2323, + "theoretical_loss": 4.579965434254028, + "tokens_seen": 135397376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843229689067202, + "loss": 3.0602, + "theoretical_loss": 4.579671285480657, + "tokens_seen": 135462912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004843129388164493, + "loss": 3.2404, + "theoretical_loss": 4.579377318803996, + "tokens_seen": 135528448 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048430290872617855, + "loss": 3.0519, + "theoretical_loss": 4.579083534023344, + "tokens_seen": 135593984 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048429287863590773, + "loss": 3.3365, + "theoretical_loss": 4.578789930938315, + "tokens_seen": 135659520 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842828485456369, + "loss": 3.2243, + "theoretical_loss": 4.5784965093488434, + "tokens_seen": 135725056 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842728184553661, + "loss": 3.1942, + "theoretical_loss": 4.578203269055179, + "tokens_seen": 135790592 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048426278836509527, + "loss": 3.4092, + "theoretical_loss": 4.577910209857889, + "tokens_seen": 135856128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048425275827482445, + "loss": 3.1487, + "theoretical_loss": 4.577617331557854, + "tokens_seen": 135921664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 86086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4546592235565186, + "objective/train/theoretical_loss": 4.577324633956271, + "objective/train/tokens_used": 156447200, + "theoretical_loss": 4.577324633956271, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842427281845537, + "loss": 3.3413, + "theoretical_loss": 4.577324633956271, + "tokens_seen": 135987200 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048423269809428287, + "loss": 3.4106, + "theoretical_loss": 4.5770321168546495, + "tokens_seen": 136052736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048422266800401205, + "loss": 3.4241, + "theoretical_loss": 4.576739780054815, + "tokens_seen": 136118272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842126379137413, + "loss": 3.254, + "theoretical_loss": 4.576447623358905, + "tokens_seen": 136183808 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004842026078234704, + "loss": 3.0941, + "theoretical_loss": 4.576155646569367, + "tokens_seen": 136249344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048419257773319965, + "loss": 3.3943, + "theoretical_loss": 4.575863849488963, + "tokens_seen": 136314880 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841825476429288, + "loss": 3.2206, + "theoretical_loss": 4.575572231920763, + "tokens_seen": 136380416 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484172517552658, + "loss": 3.1229, + "theoretical_loss": 4.575280793668151, + "tokens_seen": 136445952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841624874623872, + "loss": 2.9849, + "theoretical_loss": 4.574989534534817, + "tokens_seen": 136511488 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048415245737211637, + "loss": 3.2635, + "theoretical_loss": 4.574698454324764, + "tokens_seen": 136577024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048414242728184555, + "loss": 3.1675, + "theoretical_loss": 4.574407552842299, + "tokens_seen": 136642560 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048413239719157473, + "loss": 3.5906, + "theoretical_loss": 4.57411682989204, + "tokens_seen": 136708096 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841223671013039, + "loss": 3.2567, + "theoretical_loss": 4.57382628527891, + "tokens_seen": 136773632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048411233701103315, + "loss": 3.5297, + "theoretical_loss": 4.573535918808141, + "tokens_seen": 136839168 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004841023069207623, + "loss": 3.3214, + "theoretical_loss": 4.573245730285268, + "tokens_seen": 136904704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840922768304915, + "loss": 3.2527, + "theoretical_loss": 4.572955719516132, + "tokens_seen": 136970240 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048408224674022064, + "loss": 3.0917, + "theoretical_loss": 4.572665886306881, + "tokens_seen": 137035776 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840722166499499, + "loss": 3.0382, + "theoretical_loss": 4.572376230463965, + "tokens_seen": 137101312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048406218655967906, + "loss": 3.3253, + "theoretical_loss": 4.572086751794135, + "tokens_seen": 137166848 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048405215646940824, + "loss": 3.4228, + "theoretical_loss": 4.571797450104451, + "tokens_seen": 137232384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840421263791374, + "loss": 3.4935, + "theoretical_loss": 4.571508325202268, + "tokens_seen": 137297920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048403209628886665, + "loss": 3.2935, + "theoretical_loss": 4.5712193768952485, + "tokens_seen": 137363456 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004840220661985958, + "loss": 3.0949, + "theoretical_loss": 4.570930604991352, + "tokens_seen": 137428992 + }, + { + "epoch": 0.04, + "learning_rate": 0.000484012036108325, + "loss": 3.0731, + "theoretical_loss": 4.57064200929884, + "tokens_seen": 137494528 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048400200601805414, + "loss": 3.2526, + "theoretical_loss": 4.570353589626273, + "tokens_seen": 137560064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 86654, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.495647668838501, + "objective/train/theoretical_loss": 4.570065345782512, + "objective/train/tokens_used": 158085600, + "theoretical_loss": 4.570065345782512, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839919759277834, + "loss": 3.3585, + "theoretical_loss": 4.570065345782512, + "tokens_seen": 137625600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048398194583751256, + "loss": 3.331, + "theoretical_loss": 4.569777277576713, + "tokens_seen": 137691136 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048397191574724174, + "loss": 3.2636, + "theoretical_loss": 4.569489384818334, + "tokens_seen": 137756672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839618856569709, + "loss": 3.3772, + "theoretical_loss": 4.569201667317128, + "tokens_seen": 137822208 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839518555667001, + "loss": 3.1082, + "theoretical_loss": 4.568914124883145, + "tokens_seen": 137887744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839418254764293, + "loss": 3.567, + "theoretical_loss": 4.568626757326731, + "tokens_seen": 137953280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839317953861585, + "loss": 3.3668, + "theoretical_loss": 4.568339564458528, + "tokens_seen": 138018816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048392176529588765, + "loss": 3.2898, + "theoretical_loss": 4.568052546089472, + "tokens_seen": 138084352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004839117352056169, + "loss": 3.2154, + "theoretical_loss": 4.567765702030792, + "tokens_seen": 138149888 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048390170511534606, + "loss": 3.3787, + "theoretical_loss": 4.567479032094015, + "tokens_seen": 138215424 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048389167502507524, + "loss": 3.2051, + "theoretical_loss": 4.567192536090956, + "tokens_seen": 138280960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838816449348044, + "loss": 3.2675, + "theoretical_loss": 4.566906213833725, + "tokens_seen": 138346496 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838716148445336, + "loss": 3.4467, + "theoretical_loss": 4.566620065134725, + "tokens_seen": 138412032 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838615847542628, + "loss": 3.3219, + "theoretical_loss": 4.566334089806647, + "tokens_seen": 138477568 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483851554663992, + "loss": 3.3479, + "theoretical_loss": 4.566048287662475, + "tokens_seen": 138543104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048384152457372115, + "loss": 3.3473, + "theoretical_loss": 4.565762658515482, + "tokens_seen": 138608640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838314944834504, + "loss": 3.3177, + "theoretical_loss": 4.565477202179232, + "tokens_seen": 138674176 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004838214643931795, + "loss": 3.2072, + "theoretical_loss": 4.565191918467576, + "tokens_seen": 138739712 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048381143430290875, + "loss": 3.4187, + "theoretical_loss": 4.564906807194655, + "tokens_seen": 138805248 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048380140421263793, + "loss": 2.8605, + "theoretical_loss": 4.564621868174897, + "tokens_seen": 138870784 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837913741223671, + "loss": 3.0615, + "theoretical_loss": 4.564337101223017, + "tokens_seen": 138936320 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837813440320963, + "loss": 3.3648, + "theoretical_loss": 4.564052506154015, + "tokens_seen": 139001856 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048377131394182547, + "loss": 3.051, + "theoretical_loss": 4.563768082783182, + "tokens_seen": 139067392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048376128385155465, + "loss": 3.1675, + "theoretical_loss": 4.563483830926089, + "tokens_seen": 139132928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837512537612839, + "loss": 3.1113, + "theoretical_loss": 4.563199750398592, + "tokens_seen": 139198464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 87912, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.470696210861206, + "objective/train/theoretical_loss": 4.562915841016838, + "objective/train/tokens_used": 159724000, + "theoretical_loss": 4.562915841016838, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483741223671013, + "loss": 3.5703, + "theoretical_loss": 4.562915841016838, + "tokens_seen": 139264000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048373119358074225, + "loss": 3.2538, + "theoretical_loss": 4.5626321025972505, + "tokens_seen": 139329536 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048372116349047143, + "loss": 3.2385, + "theoretical_loss": 4.562348534956538, + "tokens_seen": 139395072 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837111334002006, + "loss": 3.4436, + "theoretical_loss": 4.562065137911691, + "tokens_seen": 139460608 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004837011033099298, + "loss": 3.1403, + "theoretical_loss": 4.561781911279987, + "tokens_seen": 139526144 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483691073219659, + "loss": 3.1669, + "theoretical_loss": 4.5614988548789785, + "tokens_seen": 139591680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048368104312938816, + "loss": 3.1656, + "theoretical_loss": 4.561215968526502, + "tokens_seen": 139657216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836710130391174, + "loss": 3.2068, + "theoretical_loss": 4.560933252040675, + "tokens_seen": 139722752 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836609829488465, + "loss": 3.2876, + "theoretical_loss": 4.56065070523989, + "tokens_seen": 139788288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048365095285857575, + "loss": 3.7268, + "theoretical_loss": 4.560368327942827, + "tokens_seen": 139853824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836409227683049, + "loss": 3.0936, + "theoretical_loss": 4.560086119968435, + "tokens_seen": 139919360 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836308926780341, + "loss": 3.362, + "theoretical_loss": 4.559804081135949, + "tokens_seen": 139984896 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836208625877633, + "loss": 3.4023, + "theoretical_loss": 4.559522211264877, + "tokens_seen": 140050432 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004836108324974925, + "loss": 3.299, + "theoretical_loss": 4.559240510175006, + "tokens_seen": 140115968 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048360080240722166, + "loss": 3.5194, + "theoretical_loss": 4.558958977686398, + "tokens_seen": 140181504 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048359077231695084, + "loss": 3.1333, + "theoretical_loss": 4.558677613619393, + "tokens_seen": 140247040 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048358074222668, + "loss": 3.1792, + "theoretical_loss": 4.558396417794603, + "tokens_seen": 140312576 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048357071213640926, + "loss": 3.2967, + "theoretical_loss": 4.558115390032916, + "tokens_seen": 140378112 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835606820461384, + "loss": 3.2453, + "theoretical_loss": 4.557834530155497, + "tokens_seen": 140443648 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835506519558676, + "loss": 3.2309, + "theoretical_loss": 4.5575538379837806, + "tokens_seen": 140509184 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835406218655968, + "loss": 3.1676, + "theoretical_loss": 4.557273313339477, + "tokens_seen": 140574720 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483530591775326, + "loss": 3.3045, + "theoretical_loss": 4.556992956044569, + "tokens_seen": 140640256 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048352056168505516, + "loss": 3.4098, + "theoretical_loss": 4.55671276592131, + "tokens_seen": 140705792 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048351053159478434, + "loss": 3.2187, + "theoretical_loss": 4.556432742792225, + "tokens_seen": 140771328 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004835005015045135, + "loss": 3.2732, + "theoretical_loss": 4.55615288648011, + "tokens_seen": 140836864 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 88907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.168332576751709, + "objective/train/theoretical_loss": 4.555873196808033, + "objective/train/tokens_used": 161362400, + "theoretical_loss": 4.555873196808033, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048349047141424276, + "loss": 3.2827, + "theoretical_loss": 4.555873196808033, + "tokens_seen": 140902400 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048348044132397194, + "loss": 3.0851, + "theoretical_loss": 4.555593673599329, + "tokens_seen": 140967936 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834704112337011, + "loss": 3.2459, + "theoretical_loss": 4.555314316677606, + "tokens_seen": 141033472 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834603811434303, + "loss": 3.1295, + "theoretical_loss": 4.555035125866738, + "tokens_seen": 141099008 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834503510531595, + "loss": 3.2718, + "theoretical_loss": 4.5547561009908675, + "tokens_seen": 141164544 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834403209628887, + "loss": 3.4012, + "theoretical_loss": 4.554477241874405, + "tokens_seen": 141230080 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048343029087261785, + "loss": 3.2815, + "theoretical_loss": 4.554198548342029, + "tokens_seen": 141295616 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004834202607823471, + "loss": 3.3752, + "theoretical_loss": 4.553920020218683, + "tokens_seen": 141361152 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048341023069207626, + "loss": 3.4304, + "theoretical_loss": 4.55364165732958, + "tokens_seen": 141426688 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048340020060180544, + "loss": 3.06, + "theoretical_loss": 4.553363459500193, + "tokens_seen": 141492224 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833901705115346, + "loss": 3.4717, + "theoretical_loss": 4.553085426556265, + "tokens_seen": 141557760 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833801404212638, + "loss": 3.2863, + "theoretical_loss": 4.552807558323801, + "tokens_seen": 141623296 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483370110330993, + "loss": 3.5702, + "theoretical_loss": 4.552529854629072, + "tokens_seen": 141688832 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833600802407222, + "loss": 3.3102, + "theoretical_loss": 4.552252315298611, + "tokens_seen": 141754368 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048335005015045135, + "loss": 3.2317, + "theoretical_loss": 4.551974940159212, + "tokens_seen": 141819904 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833400200601806, + "loss": 3.3219, + "theoretical_loss": 4.551697729037938, + "tokens_seen": 141885440 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004833299899699097, + "loss": 3.4492, + "theoretical_loss": 4.551420681762107, + "tokens_seen": 141950976 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048331995987963895, + "loss": 3.2021, + "theoretical_loss": 4.551143798159299, + "tokens_seen": 142016512 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048330992978936813, + "loss": 3.2159, + "theoretical_loss": 4.550867078057361, + "tokens_seen": 142082048 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832998996990973, + "loss": 3.2044, + "theoretical_loss": 4.550590521284393, + "tokens_seen": 142147584 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832898696088265, + "loss": 3.1647, + "theoretical_loss": 4.550314127668761, + "tokens_seen": 142213120 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048327983951855567, + "loss": 3.4173, + "theoretical_loss": 4.550037897039086, + "tokens_seen": 142278656 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048326980942828485, + "loss": 3.0888, + "theoretical_loss": 4.54976182922425, + "tokens_seen": 142344192 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832597793380141, + "loss": 3.0527, + "theoretical_loss": 4.549485924053392, + "tokens_seen": 142409728 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832497492477432, + "loss": 3.3659, + "theoretical_loss": 4.54921018135591, + "tokens_seen": 142475264 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 89401, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.601848602294922, + "objective/train/theoretical_loss": 4.548934600961459, + "objective/train/tokens_used": 163000800, + "theoretical_loss": 4.548934600961459, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048323971915747245, + "loss": 3.4687, + "theoretical_loss": 4.548934600961459, + "tokens_seen": 142540800 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048322968906720163, + "loss": 3.2753, + "theoretical_loss": 4.54865918269995, + "tokens_seen": 142606336 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004832196589769308, + "loss": 3.1228, + "theoretical_loss": 4.548383926401552, + "tokens_seen": 142671872 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048320962888666, + "loss": 3.0876, + "theoretical_loss": 4.548108831896688, + "tokens_seen": 142737408 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831995987963892, + "loss": 3.44, + "theoretical_loss": 4.547833899016037, + "tokens_seen": 142802944 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048318956870611836, + "loss": 3.0777, + "theoretical_loss": 4.547559127590535, + "tokens_seen": 142868480 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831795386158476, + "loss": 3.5091, + "theoretical_loss": 4.547284517451367, + "tokens_seen": 142934016 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831695085255767, + "loss": 3.3175, + "theoretical_loss": 4.547010068429978, + "tokens_seen": 142999552 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048315947843530595, + "loss": 3.1291, + "theoretical_loss": 4.546735780358061, + "tokens_seen": 143065088 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831494483450351, + "loss": 3.1034, + "theoretical_loss": 4.546461653067565, + "tokens_seen": 143130624 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831394182547643, + "loss": 3.0849, + "theoretical_loss": 4.54618768639069, + "tokens_seen": 143196160 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831293881644935, + "loss": 3.0665, + "theoretical_loss": 4.545913880159889, + "tokens_seen": 143261696 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004831193580742227, + "loss": 3.2806, + "theoretical_loss": 4.545640234207866, + "tokens_seen": 143327232 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048310932798395186, + "loss": 3.3249, + "theoretical_loss": 4.545366748367574, + "tokens_seen": 143392768 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048309929789368104, + "loss": 3.4583, + "theoretical_loss": 4.545093422472218, + "tokens_seen": 143458304 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830892678034102, + "loss": 3.5278, + "theoretical_loss": 4.544820256355253, + "tokens_seen": 143523840 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048307923771313946, + "loss": 3.0959, + "theoretical_loss": 4.544547249850384, + "tokens_seen": 143589376 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830692076228686, + "loss": 3.4636, + "theoretical_loss": 4.544274402791562, + "tokens_seen": 143654912 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830591775325978, + "loss": 3.1348, + "theoretical_loss": 4.544001715012989, + "tokens_seen": 143720448 + }, + { + "epoch": 0.04, + "learning_rate": 0.000483049147442327, + "loss": 3.3713, + "theoretical_loss": 4.543729186349115, + "tokens_seen": 143785984 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830391173520562, + "loss": 3.334, + "theoretical_loss": 4.5434568166346345, + "tokens_seen": 143851520 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048302908726178536, + "loss": 3.2556, + "theoretical_loss": 4.543184605704493, + "tokens_seen": 143917056 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048301905717151454, + "loss": 3.24, + "theoretical_loss": 4.542912553393879, + "tokens_seen": 143982592 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004830090270812437, + "loss": 3.0543, + "theoretical_loss": 4.542640659538229, + "tokens_seen": 144048128 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048299899699097296, + "loss": 3.3306, + "theoretical_loss": 4.542368923973226, + "tokens_seen": 144113664 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 90537, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.076249361038208, + "objective/train/theoretical_loss": 4.542097346534794, + "objective/train/tokens_used": 164639200, + "theoretical_loss": 4.542097346534794, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829889669007021, + "loss": 3.2985, + "theoretical_loss": 4.542097346534794, + "tokens_seen": 144179200 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829789368104313, + "loss": 3.2073, + "theoretical_loss": 4.541825927059106, + "tokens_seen": 144244736 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048296890672016045, + "loss": 3.3592, + "theoretical_loss": 4.541554665382575, + "tokens_seen": 144310272 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829588766298897, + "loss": 3.5993, + "theoretical_loss": 4.541283561341862, + "tokens_seen": 144375808 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048294884653961887, + "loss": 3.0916, + "theoretical_loss": 4.541012614773868, + "tokens_seen": 144441344 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048293881644934805, + "loss": 3.2114, + "theoretical_loss": 4.540741825515736, + "tokens_seen": 144506880 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048292878635907723, + "loss": 3.4082, + "theoretical_loss": 4.540471193404855, + "tokens_seen": 144572416 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048291875626880646, + "loss": 3.1695, + "theoretical_loss": 4.54020071827885, + "tokens_seen": 144637952 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004829087261785356, + "loss": 3.4008, + "theoretical_loss": 4.539930399975596, + "tokens_seen": 144703488 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828986960882648, + "loss": 3.1431, + "theoretical_loss": 4.539660238333198, + "tokens_seen": 144769024 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048288866599799395, + "loss": 3.3694, + "theoretical_loss": 4.53939023319001, + "tokens_seen": 144834560 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828786359077232, + "loss": 3.3767, + "theoretical_loss": 4.539120384384621, + "tokens_seen": 144900096 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048286860581745237, + "loss": 3.5769, + "theoretical_loss": 4.538850691755861, + "tokens_seen": 144965632 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048285857572718155, + "loss": 3.3304, + "theoretical_loss": 4.538581155142799, + "tokens_seen": 145031168 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048284854563691073, + "loss": 3.1185, + "theoretical_loss": 4.538311774384743, + "tokens_seen": 145096704 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828385155466399, + "loss": 3.4686, + "theoretical_loss": 4.538042549321238, + "tokens_seen": 145162240 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004828284854563691, + "loss": 3.1808, + "theoretical_loss": 4.537773479792067, + "tokens_seen": 145227776 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048281845536609833, + "loss": 3.2259, + "theoretical_loss": 4.537504565637248, + "tokens_seen": 145293312 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048280842527582746, + "loss": 3.3492, + "theoretical_loss": 4.537235806697041, + "tokens_seen": 145358848 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827983951855567, + "loss": 3.4474, + "theoretical_loss": 4.536967202811935, + "tokens_seen": 145424384 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827883650952858, + "loss": 3.2921, + "theoretical_loss": 4.536698753822661, + "tokens_seen": 145489920 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048277833500501505, + "loss": 3.4976, + "theoretical_loss": 4.53643045957018, + "tokens_seen": 145555456 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048276830491474423, + "loss": 3.4497, + "theoretical_loss": 4.536162319895695, + "tokens_seen": 145620992 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827582748244734, + "loss": 3.4119, + "theoretical_loss": 4.535894334640634, + "tokens_seen": 145686528 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827482447342026, + "loss": 3.0995, + "theoretical_loss": 4.535626503646666, + "tokens_seen": 145752064 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 91176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9223389625549316, + "objective/train/theoretical_loss": 4.535358826755691, + "objective/train/tokens_used": 166277600, + "theoretical_loss": 4.535358826755691, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048273821464393183, + "loss": 3.2178, + "theoretical_loss": 4.535358826755691, + "tokens_seen": 145817600 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482728184553661, + "loss": 3.4704, + "theoretical_loss": 4.535091303809842, + "tokens_seen": 145883136 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827181544633902, + "loss": 3.2936, + "theoretical_loss": 4.534823934651484, + "tokens_seen": 145948672 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004827081243731194, + "loss": 3.1424, + "theoretical_loss": 4.5345567191232155, + "tokens_seen": 146014208 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048269809428284856, + "loss": 3.3168, + "theoretical_loss": 4.534289657067865, + "tokens_seen": 146079744 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826880641925778, + "loss": 2.9011, + "theoretical_loss": 4.5340227483284945, + "tokens_seen": 146145280 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826780341023069, + "loss": 3.2725, + "theoretical_loss": 4.533755992748395, + "tokens_seen": 146210816 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048266800401203615, + "loss": 3.2624, + "theoretical_loss": 4.533489390171088, + "tokens_seen": 146276352 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826579739217653, + "loss": 3.4202, + "theoretical_loss": 4.533222940440325, + "tokens_seen": 146341888 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826479438314945, + "loss": 3.1386, + "theoretical_loss": 4.532956643400087, + "tokens_seen": 146407424 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826379137412237, + "loss": 3.3923, + "theoretical_loss": 4.532690498894585, + "tokens_seen": 146472960 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004826278836509529, + "loss": 3.4023, + "theoretical_loss": 4.5324245067682565, + "tokens_seen": 146538496 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048261785356068206, + "loss": 3.1156, + "theoretical_loss": 4.532158666865769, + "tokens_seen": 146604032 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048260782347041124, + "loss": 3.1851, + "theoretical_loss": 4.5318929790320155, + "tokens_seen": 146669568 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825977933801404, + "loss": 3.0943, + "theoretical_loss": 4.53162744311212, + "tokens_seen": 146735104 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048258776328986966, + "loss": 3.392, + "theoretical_loss": 4.53136205895143, + "tokens_seen": 146800640 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825777331995988, + "loss": 3.4047, + "theoretical_loss": 4.531096826395519, + "tokens_seen": 146866176 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482567703109328, + "loss": 3.2556, + "theoretical_loss": 4.5308317452901905, + "tokens_seen": 146931712 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825576730190572, + "loss": 3.1769, + "theoretical_loss": 4.530566815481469, + "tokens_seen": 146997248 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825476429287864, + "loss": 3.2533, + "theoretical_loss": 4.530302036815607, + "tokens_seen": 147062784 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048253761283851556, + "loss": 3.3405, + "theoretical_loss": 4.53003740913908, + "tokens_seen": 147128320 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048252758274824474, + "loss": 3.3032, + "theoretical_loss": 4.529772932298588, + "tokens_seen": 147193856 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004825175526579739, + "loss": 3.4738, + "theoretical_loss": 4.529508606141057, + "tokens_seen": 147259392 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048250752256770316, + "loss": 3.2304, + "theoretical_loss": 4.529244430513632, + "tokens_seen": 147324928 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824974924774323, + "loss": 3.2967, + "theoretical_loss": 4.528980405263686, + "tokens_seen": 147390464 + }, + { + "epoch": 0.04, + "objective/train/docs_used": 92532, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4305601119995117, + "objective/train/theoretical_loss": 4.528716530238812, + "objective/train/tokens_used": 167916000, + "theoretical_loss": 4.528716530238812, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824874623871615, + "loss": 3.4659, + "theoretical_loss": 4.528716530238812, + "tokens_seen": 147456000 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048247743229689065, + "loss": 3.2645, + "theoretical_loss": 4.528452805286824, + "tokens_seen": 147521536 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824674022066199, + "loss": 3.2269, + "theoretical_loss": 4.528189230255759, + "tokens_seen": 147587072 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048245737211634907, + "loss": 3.2869, + "theoretical_loss": 4.527925804993875, + "tokens_seen": 147652608 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048244734202607825, + "loss": 3.3874, + "theoretical_loss": 4.5276625293496515, + "tokens_seen": 147718144 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048243731193580743, + "loss": 3.1961, + "theoretical_loss": 4.527399403171788, + "tokens_seen": 147783680 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048242728184553666, + "loss": 3.3541, + "theoretical_loss": 4.5271364263092035, + "tokens_seen": 147849216 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004824172517552658, + "loss": 3.2284, + "theoretical_loss": 4.526873598611038, + "tokens_seen": 147914752 + }, + { + "epoch": 0.04, + "learning_rate": 0.000482407221664995, + "loss": 3.1859, + "theoretical_loss": 4.52661091992665, + "tokens_seen": 147980288 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048239719157472415, + "loss": 3.3306, + "theoretical_loss": 4.526348390105614, + "tokens_seen": 148045824 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823871614844534, + "loss": 3.0245, + "theoretical_loss": 4.526086008997728, + "tokens_seen": 148111360 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048237713139418257, + "loss": 3.1419, + "theoretical_loss": 4.525823776453004, + "tokens_seen": 148176896 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048236710130391175, + "loss": 3.0672, + "theoretical_loss": 4.525561692321673, + "tokens_seen": 148242432 + }, + { + "epoch": 0.04, + "learning_rate": 0.00048235707121364093, + "loss": 3.259, + "theoretical_loss": 4.525299756454182, + "tokens_seen": 148307968 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823470411233701, + "loss": 3.3995, + "theoretical_loss": 4.525037968701198, + "tokens_seen": 148373504 + }, + { + "epoch": 0.04, + "learning_rate": 0.0004823370110330993, + "loss": 3.1846, + "theoretical_loss": 4.5247763289136, + "tokens_seen": 148439040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048232698094282853, + "loss": 3.3233, + "theoretical_loss": 4.524514836942485, + "tokens_seen": 148504576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048231695085255766, + "loss": 3.1244, + "theoretical_loss": 4.524253492639165, + "tokens_seen": 148570112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004823069207622869, + "loss": 3.1294, + "theoretical_loss": 4.523992295855169, + "tokens_seen": 148635648 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482296890672016, + "loss": 3.4438, + "theoretical_loss": 4.523731246442237, + "tokens_seen": 148701184 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048228686058174525, + "loss": 3.2735, + "theoretical_loss": 4.523470344252326, + "tokens_seen": 148766720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048227683049147443, + "loss": 3.2357, + "theoretical_loss": 4.523209589137606, + "tokens_seen": 148832256 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822668004012036, + "loss": 3.3941, + "theoretical_loss": 4.522948980950461, + "tokens_seen": 148897792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822567703109328, + "loss": 3.0532, + "theoretical_loss": 4.522688519543487, + "tokens_seen": 148963328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048224674022066203, + "loss": 3.1432, + "theoretical_loss": 4.5224282047694935, + "tokens_seen": 149028864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 93022, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5269510746002197, + "objective/train/theoretical_loss": 4.522168036481502, + "objective/train/tokens_used": 169554400, + "theoretical_loss": 4.522168036481502, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048223671013039116, + "loss": 3.5101, + "theoretical_loss": 4.522168036481502, + "tokens_seen": 149094400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822266800401204, + "loss": 3.2561, + "theoretical_loss": 4.521908014532746, + "tokens_seen": 149159936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004822166499498495, + "loss": 3.3344, + "theoretical_loss": 4.52164813877667, + "tokens_seen": 149225472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048220661985957876, + "loss": 3.2209, + "theoretical_loss": 4.52138840906693, + "tokens_seen": 149291008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048219658976930794, + "loss": 3.1369, + "theoretical_loss": 4.521128825257392, + "tokens_seen": 149356544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821865596790371, + "loss": 3.1751, + "theoretical_loss": 4.520869387202134, + "tokens_seen": 149422080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821765295887663, + "loss": 2.9797, + "theoretical_loss": 4.52061009475544, + "tokens_seen": 149487616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821664994984955, + "loss": 3.196, + "theoretical_loss": 4.52035094777181, + "tokens_seen": 149553152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048215646940822466, + "loss": 3.3068, + "theoretical_loss": 4.520091946105946, + "tokens_seen": 149618688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821464393179539, + "loss": 3.4288, + "theoretical_loss": 4.519833089612764, + "tokens_seen": 149684224 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482136409227683, + "loss": 3.2192, + "theoretical_loss": 4.519574378147384, + "tokens_seen": 149749760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048212637913741226, + "loss": 3.291, + "theoretical_loss": 4.519315811565137, + "tokens_seen": 149815296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821163490471414, + "loss": 3.3752, + "theoretical_loss": 4.51905738972156, + "tokens_seen": 149880832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004821063189568706, + "loss": 3.6035, + "theoretical_loss": 4.518799112472401, + "tokens_seen": 149946368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820962888665998, + "loss": 3.5239, + "theoretical_loss": 4.518540979673606, + "tokens_seen": 150011904 + }, + { + "epoch": 0.05, + "learning_rate": 0.000482086258776329, + "loss": 3.1651, + "theoretical_loss": 4.518282991181337, + "tokens_seen": 150077440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048207622868605816, + "loss": 3.1406, + "theoretical_loss": 4.518025146851956, + "tokens_seen": 150142976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820661985957874, + "loss": 3.2538, + "theoretical_loss": 4.517767446542033, + "tokens_seen": 150208512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048205616850551653, + "loss": 3.1976, + "theoretical_loss": 4.517509890108343, + "tokens_seen": 150274048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048204613841524576, + "loss": 3.1097, + "theoretical_loss": 4.5172524774078635, + "tokens_seen": 150339584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820361083249749, + "loss": 3.3081, + "theoretical_loss": 4.5169952082977805, + "tokens_seen": 150405120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820260782347041, + "loss": 3.2403, + "theoretical_loss": 4.51673808263548, + "tokens_seen": 150470656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820160481444333, + "loss": 3.0648, + "theoretical_loss": 4.516481100278555, + "tokens_seen": 150536192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004820060180541625, + "loss": 3.1256, + "theoretical_loss": 4.516224261084799, + "tokens_seen": 150601728 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048199598796389167, + "loss": 3.0304, + "theoretical_loss": 4.515967564912211, + "tokens_seen": 150667264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 94173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2401325702667236, + "objective/train/theoretical_loss": 4.515711011618992, + "objective/train/tokens_used": 171192800, + "theoretical_loss": 4.515711011618992, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048198595787362085, + "loss": 3.2269, + "theoretical_loss": 4.515711011618992, + "tokens_seen": 150732800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004819759277833501, + "loss": 3.1091, + "theoretical_loss": 4.515454601063542, + "tokens_seen": 150798336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048196589769307927, + "loss": 3.2182, + "theoretical_loss": 4.5151983331044665, + "tokens_seen": 150863872 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048195586760280845, + "loss": 3.2107, + "theoretical_loss": 4.514942207600572, + "tokens_seen": 150929408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048194583751253763, + "loss": 3.1827, + "theoretical_loss": 4.514686224410863, + "tokens_seen": 150994944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048193580742226686, + "loss": 3.3203, + "theoretical_loss": 4.514430383394549, + "tokens_seen": 151060480 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481925777331996, + "loss": 3.0874, + "theoretical_loss": 4.514174684411037, + "tokens_seen": 151126016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004819157472417252, + "loss": 3.2352, + "theoretical_loss": 4.513919127319935, + "tokens_seen": 151191552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048190571715145435, + "loss": 3.3179, + "theoretical_loss": 4.513663711981049, + "tokens_seen": 151257088 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818956870611836, + "loss": 3.266, + "theoretical_loss": 4.513408438254386, + "tokens_seen": 151322624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048188565697091277, + "loss": 3.1682, + "theoretical_loss": 4.5131533060001505, + "tokens_seen": 151388160 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048187562688064195, + "loss": 3.2016, + "theoretical_loss": 4.512898315078747, + "tokens_seen": 151453696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048186559679037113, + "loss": 3.2221, + "theoretical_loss": 4.512643465350775, + "tokens_seen": 151519232 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818555667001003, + "loss": 3.394, + "theoretical_loss": 4.512388756677037, + "tokens_seen": 151584768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818455366098295, + "loss": 3.3002, + "theoretical_loss": 4.5121341889185285, + "tokens_seen": 151650304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048183550651955873, + "loss": 3.131, + "theoretical_loss": 4.511879761936442, + "tokens_seen": 151715840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048182547642928786, + "loss": 3.3703, + "theoretical_loss": 4.511625475592169, + "tokens_seen": 151781376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818154463390171, + "loss": 3.2669, + "theoretical_loss": 4.511371329747295, + "tokens_seen": 151846912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004818054162487462, + "loss": 3.0424, + "theoretical_loss": 4.511117324263603, + "tokens_seen": 151912448 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048179538615847545, + "loss": 3.6565, + "theoretical_loss": 4.5108634590030725, + "tokens_seen": 151977984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048178535606820463, + "loss": 3.411, + "theoretical_loss": 4.510609733827873, + "tokens_seen": 152043520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817753259779338, + "loss": 3.1894, + "theoretical_loss": 4.5103561486003745, + "tokens_seen": 152109056 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481765295887663, + "loss": 3.053, + "theoretical_loss": 4.51010270318314, + "tokens_seen": 152174592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048175526579739223, + "loss": 3.1504, + "theoretical_loss": 4.509849397438925, + "tokens_seen": 152240128 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048174523570712136, + "loss": 3.2262, + "theoretical_loss": 4.509596231230679, + "tokens_seen": 152305664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 94867, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.429643392562866, + "objective/train/theoretical_loss": 4.509343204421548, + "objective/train/tokens_used": 172831200, + "theoretical_loss": 4.509343204421548, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817352056168506, + "loss": 3.4209, + "theoretical_loss": 4.509343204421548, + "tokens_seen": 152371200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004817251755265797, + "loss": 3.2597, + "theoretical_loss": 4.509090316874865, + "tokens_seen": 152436736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048171514543630896, + "loss": 3.2857, + "theoretical_loss": 4.508837568454162, + "tokens_seen": 152502272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048170511534603814, + "loss": 3.5512, + "theoretical_loss": 4.508584959023159, + "tokens_seen": 152567808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816950852557673, + "loss": 3.3878, + "theoretical_loss": 4.50833248844577, + "tokens_seen": 152633344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816850551654965, + "loss": 3.3808, + "theoretical_loss": 4.5080801565861, + "tokens_seen": 152698880 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816750250752257, + "loss": 3.1163, + "theoretical_loss": 4.507827963308447, + "tokens_seen": 152764416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048166499498495486, + "loss": 3.2056, + "theoretical_loss": 4.507575908477296, + "tokens_seen": 152829952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816549648946841, + "loss": 3.1235, + "theoretical_loss": 4.507323991957325, + "tokens_seen": 152895488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816449348044132, + "loss": 3.439, + "theoretical_loss": 4.507072213613402, + "tokens_seen": 152961024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048163490471414246, + "loss": 3.2998, + "theoretical_loss": 4.506820573310586, + "tokens_seen": 153026560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816248746238716, + "loss": 3.5354, + "theoretical_loss": 4.506569070914123, + "tokens_seen": 153092096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004816148445336008, + "loss": 3.1693, + "theoretical_loss": 4.506317706289451, + "tokens_seen": 153157632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048160481444333, + "loss": 3.3531, + "theoretical_loss": 4.506066479302193, + "tokens_seen": 153223168 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815947843530592, + "loss": 3.3859, + "theoretical_loss": 4.505815389818165, + "tokens_seen": 153288704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048158475426278837, + "loss": 3.2548, + "theoretical_loss": 4.505564437703368, + "tokens_seen": 153354240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815747241725176, + "loss": 3.1131, + "theoretical_loss": 4.5053136228239925, + "tokens_seen": 153419776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048156469408224673, + "loss": 3.3206, + "theoretical_loss": 4.505062945046413, + "tokens_seen": 153485312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048155466399197596, + "loss": 3.3988, + "theoretical_loss": 4.504812404237197, + "tokens_seen": 153550848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815446339017051, + "loss": 3.1752, + "theoretical_loss": 4.504562000263093, + "tokens_seen": 153616384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815346038114343, + "loss": 3.1504, + "theoretical_loss": 4.504311732991039, + "tokens_seen": 153681920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815245737211635, + "loss": 3.0018, + "theoretical_loss": 4.504061602288159, + "tokens_seen": 153747456 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004815145436308927, + "loss": 2.9879, + "theoretical_loss": 4.503811608021761, + "tokens_seen": 153812992 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048150451354062187, + "loss": 3.1938, + "theoretical_loss": 4.50356175005934, + "tokens_seen": 153878528 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048149448345035105, + "loss": 3.5199, + "theoretical_loss": 4.503312028268576, + "tokens_seen": 153944064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 95853, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.367169141769409, + "objective/train/theoretical_loss": 4.503062442517334, + "objective/train/tokens_used": 174469600, + "theoretical_loss": 4.503062442517334, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048148445336008023, + "loss": 3.257, + "theoretical_loss": 4.503062442517334, + "tokens_seen": 154009600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048147442326980947, + "loss": 3.4149, + "theoretical_loss": 4.502812992673661, + "tokens_seen": 154075136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814643931795386, + "loss": 3.3831, + "theoretical_loss": 4.50256367860579, + "tokens_seen": 154140672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048145436308926783, + "loss": 3.2127, + "theoretical_loss": 4.502314500182138, + "tokens_seen": 154206208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048144433299899696, + "loss": 3.0253, + "theoretical_loss": 4.502065457271304, + "tokens_seen": 154271744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004814343029087262, + "loss": 3.522, + "theoretical_loss": 4.501816549742072, + "tokens_seen": 154337280 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048142427281845537, + "loss": 3.031, + "theoretical_loss": 4.501567777463406, + "tokens_seen": 154402816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048141424272818455, + "loss": 3.33, + "theoretical_loss": 4.501319140304456, + "tokens_seen": 154468352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048140421263791373, + "loss": 3.0943, + "theoretical_loss": 4.50107063813455, + "tokens_seen": 154533888 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048139418254764297, + "loss": 2.9423, + "theoretical_loss": 4.5008222708232, + "tokens_seen": 154599424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813841524573721, + "loss": 3.2762, + "theoretical_loss": 4.500574038240099, + "tokens_seen": 154664960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048137412236710133, + "loss": 3.1341, + "theoretical_loss": 4.500325940255123, + "tokens_seen": 154730496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048136409227683046, + "loss": 3.323, + "theoretical_loss": 4.500077976738324, + "tokens_seen": 154796032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813540621865597, + "loss": 3.2968, + "theoretical_loss": 4.499830147559939, + "tokens_seen": 154861568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813440320962889, + "loss": 3.3287, + "theoretical_loss": 4.499582452590383, + "tokens_seen": 154927104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048133400200601806, + "loss": 3.4478, + "theoretical_loss": 4.499334891700251, + "tokens_seen": 154992640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048132397191574724, + "loss": 3.1801, + "theoretical_loss": 4.499087464760317, + "tokens_seen": 155058176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813139418254764, + "loss": 3.0457, + "theoretical_loss": 4.498840171641534, + "tokens_seen": 155123712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004813039117352056, + "loss": 2.9473, + "theoretical_loss": 4.498593012215036, + "tokens_seen": 155189248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048129388164493483, + "loss": 3.1367, + "theoretical_loss": 4.498345986352133, + "tokens_seen": 155254784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048128385155466396, + "loss": 3.2789, + "theoretical_loss": 4.4980990939243135, + "tokens_seen": 155320320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812738214643932, + "loss": 3.265, + "theoretical_loss": 4.497852334803245, + "tokens_seen": 155385856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812637913741223, + "loss": 3.2818, + "theoretical_loss": 4.497605708860771, + "tokens_seen": 155451392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048125376128385156, + "loss": 3.148, + "theoretical_loss": 4.497359215968914, + "tokens_seen": 155516928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048124373119358074, + "loss": 3.1139, + "theoretical_loss": 4.497112855999871, + "tokens_seen": 155582464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 96994, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.577995777130127, + "objective/train/theoretical_loss": 4.496866628826018, + "objective/train/tokens_used": 176108000, + "theoretical_loss": 4.496866628826018, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812337011033099, + "loss": 3.366, + "theoretical_loss": 4.496866628826018, + "tokens_seen": 155648000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048122367101303916, + "loss": 3.2348, + "theoretical_loss": 4.496620534319904, + "tokens_seen": 155713536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048121364092276834, + "loss": 3.1337, + "theoretical_loss": 4.496374572354256, + "tokens_seen": 155779072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004812036108324975, + "loss": 3.4091, + "theoretical_loss": 4.496128742801979, + "tokens_seen": 155844608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811935807422267, + "loss": 3.3832, + "theoretical_loss": 4.495883045536148, + "tokens_seen": 155910144 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811835506519559, + "loss": 3.1527, + "theoretical_loss": 4.495637480430018, + "tokens_seen": 155975680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048117352056168506, + "loss": 3.3667, + "theoretical_loss": 4.495392047357015, + "tokens_seen": 156041216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811634904714143, + "loss": 3.1819, + "theoretical_loss": 4.4951467461907395, + "tokens_seen": 156106752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811534603811434, + "loss": 3.2472, + "theoretical_loss": 4.494901576804969, + "tokens_seen": 156172288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048114343029087266, + "loss": 3.3371, + "theoretical_loss": 4.4946565390736515, + "tokens_seen": 156237824 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811334002006018, + "loss": 3.3338, + "theoretical_loss": 4.494411632870911, + "tokens_seen": 156303360 + }, + { + "epoch": 0.05, + "learning_rate": 0.000481123370110331, + "loss": 3.3003, + "theoretical_loss": 4.494166858071041, + "tokens_seen": 156368896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811133400200602, + "loss": 3.0684, + "theoretical_loss": 4.493922214548512, + "tokens_seen": 156434432 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004811033099297894, + "loss": 3.1874, + "theoretical_loss": 4.493677702177964, + "tokens_seen": 156499968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048109327983951857, + "loss": 3.2209, + "theoretical_loss": 4.493433320834209, + "tokens_seen": 156565504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810832497492478, + "loss": 3.3108, + "theoretical_loss": 4.493189070392235, + "tokens_seen": 156631040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048107321965897693, + "loss": 3.0176, + "theoretical_loss": 4.492944950727196, + "tokens_seen": 156696576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048106318956870616, + "loss": 3.0247, + "theoretical_loss": 4.492700961714419, + "tokens_seen": 156762112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810531594784353, + "loss": 3.1494, + "theoretical_loss": 4.492457103229405, + "tokens_seen": 156827648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810431293881645, + "loss": 3.4656, + "theoretical_loss": 4.492213375147822, + "tokens_seen": 156893184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810330992978937, + "loss": 3.1795, + "theoretical_loss": 4.491969777345509, + "tokens_seen": 156958720 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004810230692076229, + "loss": 3.2572, + "theoretical_loss": 4.491726309698475, + "tokens_seen": 157024256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048101303911735207, + "loss": 3.074, + "theoretical_loss": 4.491482972082901, + "tokens_seen": 157089792 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048100300902708125, + "loss": 3.3692, + "theoretical_loss": 4.491239764375134, + "tokens_seen": 157155328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048099297893681043, + "loss": 3.166, + "theoretical_loss": 4.490996686451692, + "tokens_seen": 157220864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 97532, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0817697048187256, + "objective/train/theoretical_loss": 4.4907537381892615, + "objective/train/tokens_used": 177746400, + "theoretical_loss": 4.4907537381892615, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048098294884653967, + "loss": 3.0902, + "theoretical_loss": 4.4907537381892615, + "tokens_seen": 157286400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809729187562688, + "loss": 3.3092, + "theoretical_loss": 4.490510919464697, + "tokens_seen": 157351936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048096288866599803, + "loss": 3.1099, + "theoretical_loss": 4.4902682301550225, + "tokens_seen": 157417472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048095285857572716, + "loss": 2.9646, + "theoretical_loss": 4.4900256701374275, + "tokens_seen": 157483008 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004809428284854564, + "loss": 3.1576, + "theoretical_loss": 4.489783239289271, + "tokens_seen": 157548544 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048093279839518557, + "loss": 3.1089, + "theoretical_loss": 4.48954093748808, + "tokens_seen": 157614080 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048092276830491475, + "loss": 3.4034, + "theoretical_loss": 4.489298764611546, + "tokens_seen": 157679616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048091273821464393, + "loss": 2.9096, + "theoretical_loss": 4.48905672053753, + "tokens_seen": 157745152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048090270812437317, + "loss": 3.1601, + "theoretical_loss": 4.488814805144055, + "tokens_seen": 157810688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808926780341023, + "loss": 3.1648, + "theoretical_loss": 4.4885730183093155, + "tokens_seen": 157876224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048088264794383153, + "loss": 3.3537, + "theoretical_loss": 4.488331359911669, + "tokens_seen": 157941760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048087261785356066, + "loss": 3.1861, + "theoretical_loss": 4.488089829829638, + "tokens_seen": 158007296 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808625877632899, + "loss": 3.1146, + "theoretical_loss": 4.487848427941911, + "tokens_seen": 158072832 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808525576730191, + "loss": 3.4209, + "theoretical_loss": 4.487607154127343, + "tokens_seen": 158138368 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048084252758274826, + "loss": 3.4348, + "theoretical_loss": 4.487366008264949, + "tokens_seen": 158203904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048083249749247744, + "loss": 3.3548, + "theoretical_loss": 4.487124990233915, + "tokens_seen": 158269440 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808224674022066, + "loss": 3.2445, + "theoretical_loss": 4.486884099913585, + "tokens_seen": 158334976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004808124373119358, + "loss": 3.5389, + "theoretical_loss": 4.486643337183471, + "tokens_seen": 158400512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048080240722166503, + "loss": 2.9913, + "theoretical_loss": 4.486402701923245, + "tokens_seen": 158466048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048079237713139416, + "loss": 3.1903, + "theoretical_loss": 4.486162194012745, + "tokens_seen": 158531584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807823470411234, + "loss": 3.3898, + "theoretical_loss": 4.48592181333197, + "tokens_seen": 158597120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807723169508525, + "loss": 3.134, + "theoretical_loss": 4.485681559761082, + "tokens_seen": 158662656 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048076228686058176, + "loss": 3.1577, + "theoretical_loss": 4.485441433180408, + "tokens_seen": 158728192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048075225677031094, + "loss": 3.0772, + "theoretical_loss": 4.485201433470432, + "tokens_seen": 158793728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807422266800401, + "loss": 3.3051, + "theoretical_loss": 4.484961560511804, + "tokens_seen": 158859264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 98333, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.562363386154175, + "objective/train/theoretical_loss": 4.484721814185333, + "objective/train/tokens_used": 179384800, + "theoretical_loss": 4.484721814185333, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807321965897693, + "loss": 3.3407, + "theoretical_loss": 4.484721814185333, + "tokens_seen": 158924800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048072216649949854, + "loss": 3.439, + "theoretical_loss": 4.48448219437199, + "tokens_seen": 158990336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048071213640922766, + "loss": 3.5068, + "theoretical_loss": 4.484242700952908, + "tokens_seen": 159055872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004807021063189569, + "loss": 3.4014, + "theoretical_loss": 4.48400333380938, + "tokens_seen": 159121408 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480692076228686, + "loss": 3.3618, + "theoretical_loss": 4.483764092822857, + "tokens_seen": 159186944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048068204613841526, + "loss": 3.2863, + "theoretical_loss": 4.483524977874953, + "tokens_seen": 159252480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048067201604814444, + "loss": 3.2132, + "theoretical_loss": 4.48328598884744, + "tokens_seen": 159318016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806619859578736, + "loss": 3.2951, + "theoretical_loss": 4.483047125622251, + "tokens_seen": 159383552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806519558676028, + "loss": 3.3459, + "theoretical_loss": 4.482808388081477, + "tokens_seen": 159449088 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480641925777332, + "loss": 3.1496, + "theoretical_loss": 4.482569776107368, + "tokens_seen": 159514624 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048063189568706117, + "loss": 3.2453, + "theoretical_loss": 4.482331289582334, + "tokens_seen": 159580160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004806218655967904, + "loss": 3.1819, + "theoretical_loss": 4.482092928388941, + "tokens_seen": 159645696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048061183550651953, + "loss": 3.2609, + "theoretical_loss": 4.4818546924099145, + "tokens_seen": 159711232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048060180541624877, + "loss": 3.2747, + "theoretical_loss": 4.481616581528138, + "tokens_seen": 159776768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048059177532597795, + "loss": 3.1228, + "theoretical_loss": 4.481378595626652, + "tokens_seen": 159842304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048058174523570713, + "loss": 3.0437, + "theoretical_loss": 4.481140734588655, + "tokens_seen": 159907840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805717151454363, + "loss": 3.3693, + "theoretical_loss": 4.4809029982975, + "tokens_seen": 159973376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805616850551655, + "loss": 3.4456, + "theoretical_loss": 4.480665386636701, + "tokens_seen": 160038912 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048055165496489467, + "loss": 3.1342, + "theoretical_loss": 4.480427899489925, + "tokens_seen": 160104448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805416248746239, + "loss": 2.9679, + "theoretical_loss": 4.480190536740995, + "tokens_seen": 160169984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048053159478435303, + "loss": 3.4332, + "theoretical_loss": 4.479953298273893, + "tokens_seen": 160235520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048052156469408227, + "loss": 3.2521, + "theoretical_loss": 4.479716183972753, + "tokens_seen": 160301056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004805115346038114, + "loss": 3.5047, + "theoretical_loss": 4.4794791937218665, + "tokens_seen": 160366592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048050150451354063, + "loss": 3.3783, + "theoretical_loss": 4.47924232740568, + "tokens_seen": 160432128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804914744232698, + "loss": 3.3839, + "theoretical_loss": 4.479005584908793, + "tokens_seen": 160497664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 99019, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4908287525177, + "objective/train/theoretical_loss": 4.478768966115963, + "objective/train/tokens_used": 181023200, + "theoretical_loss": 4.478768966115963, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480481444332999, + "loss": 3.1208, + "theoretical_loss": 4.478768966115963, + "tokens_seen": 160563200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048047141424272823, + "loss": 3.2379, + "theoretical_loss": 4.478532470912096, + "tokens_seen": 160628736 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048046138415245736, + "loss": 3.0995, + "theoretical_loss": 4.478296099182261, + "tokens_seen": 160694272 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804513540621866, + "loss": 3.2414, + "theoretical_loss": 4.47805985081167, + "tokens_seen": 160759808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048044132397191577, + "loss": 3.1472, + "theoretical_loss": 4.477823725685695, + "tokens_seen": 160825344 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048043129388164495, + "loss": 3.2584, + "theoretical_loss": 4.477587723689862, + "tokens_seen": 160890880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048042126379137413, + "loss": 3.2151, + "theoretical_loss": 4.477351844709846, + "tokens_seen": 160956416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048041123370110337, + "loss": 3.464, + "theoretical_loss": 4.477116088631476, + "tokens_seen": 161021952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004804012036108325, + "loss": 3.2482, + "theoretical_loss": 4.476880455340735, + "tokens_seen": 161087488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048039117352056173, + "loss": 3.347, + "theoretical_loss": 4.4766449447237555, + "tokens_seen": 161153024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048038114343029086, + "loss": 3.2773, + "theoretical_loss": 4.476409556666825, + "tokens_seen": 161218560 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803711133400201, + "loss": 3.0332, + "theoretical_loss": 4.476174291056379, + "tokens_seen": 161284096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803610832497493, + "loss": 3.2546, + "theoretical_loss": 4.475939147779008, + "tokens_seen": 161349632 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048035105315947846, + "loss": 3.0797, + "theoretical_loss": 4.47570412672145, + "tokens_seen": 161415168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048034102306920764, + "loss": 3.1193, + "theoretical_loss": 4.475469227770596, + "tokens_seen": 161480704 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004803309929789368, + "loss": 3.222, + "theoretical_loss": 4.475234450813487, + "tokens_seen": 161546240 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480320962888666, + "loss": 3.3821, + "theoretical_loss": 4.474999795737315, + "tokens_seen": 161611776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048031093279839523, + "loss": 3.17, + "theoretical_loss": 4.47476526242942, + "tokens_seen": 161677312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048030090270812436, + "loss": 3.0239, + "theoretical_loss": 4.474530850777294, + "tokens_seen": 161742848 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802908726178536, + "loss": 3.2871, + "theoretical_loss": 4.474296560668577, + "tokens_seen": 161808384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802808425275827, + "loss": 3.0498, + "theoretical_loss": 4.474062391991058, + "tokens_seen": 161873920 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048027081243731196, + "loss": 3.0457, + "theoretical_loss": 4.473828344632677, + "tokens_seen": 161939456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048026078234704114, + "loss": 3.114, + "theoretical_loss": 4.473594418481522, + "tokens_seen": 162004992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802507522567703, + "loss": 3.3295, + "theoretical_loss": 4.473360613425827, + "tokens_seen": 162070528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802407221664995, + "loss": 3.046, + "theoretical_loss": 4.473126929353977, + "tokens_seen": 162136064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 100000, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.690746545791626, + "objective/train/theoretical_loss": 4.472893366154503, + "objective/train/tokens_used": 182661600, + "theoretical_loss": 4.472893366154503, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048023069207622874, + "loss": 2.9009, + "theoretical_loss": 4.472893366154503, + "tokens_seen": 162201600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048022066198595786, + "loss": 3.0755, + "theoretical_loss": 4.472659923716087, + "tokens_seen": 162267136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004802106318956871, + "loss": 3.0381, + "theoretical_loss": 4.472426601927554, + "tokens_seen": 162332672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048020060180541623, + "loss": 3.0383, + "theoretical_loss": 4.472193400677878, + "tokens_seen": 162398208 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048019057171514546, + "loss": 3.207, + "theoretical_loss": 4.471960319856182, + "tokens_seen": 162463744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048018054162487464, + "loss": 3.0581, + "theoretical_loss": 4.471727359351734, + "tokens_seen": 162529280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801705115346038, + "loss": 3.323, + "theoretical_loss": 4.4714945190539455, + "tokens_seen": 162594816 + }, + { + "epoch": 0.05, + "learning_rate": 0.000480160481444333, + "loss": 3.3514, + "theoretical_loss": 4.47126179885238, + "tokens_seen": 162660352 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801504513540622, + "loss": 3.0515, + "theoretical_loss": 4.471029198636741, + "tokens_seen": 162725888 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048014042126379137, + "loss": 2.9545, + "theoretical_loss": 4.470796718296884, + "tokens_seen": 162791424 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004801303911735206, + "loss": 3.0233, + "theoretical_loss": 4.470564357722804, + "tokens_seen": 162856960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048012036108324973, + "loss": 3.258, + "theoretical_loss": 4.470332116804644, + "tokens_seen": 162922496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048011033099297897, + "loss": 3.2388, + "theoretical_loss": 4.470099995432692, + "tokens_seen": 162988032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048010030090270815, + "loss": 3.0379, + "theoretical_loss": 4.469867993497379, + "tokens_seen": 163053568 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048009027081243733, + "loss": 3.0891, + "theoretical_loss": 4.469636110889283, + "tokens_seen": 163119104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800802407221665, + "loss": 3.2988, + "theoretical_loss": 4.4694043474991245, + "tokens_seen": 163184640 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800702106318957, + "loss": 3.0297, + "theoretical_loss": 4.469172703217767, + "tokens_seen": 163250176 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048006018054162487, + "loss": 3.0857, + "theoretical_loss": 4.46894117793622, + "tokens_seen": 163315712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800501504513541, + "loss": 3.1744, + "theoretical_loss": 4.468709771545634, + "tokens_seen": 163381248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048004012036108323, + "loss": 3.0103, + "theoretical_loss": 4.4684784839373055, + "tokens_seen": 163446784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048003009027081247, + "loss": 3.2331, + "theoretical_loss": 4.468247315002671, + "tokens_seen": 163512320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004800200601805416, + "loss": 3.1392, + "theoretical_loss": 4.468016264633311, + "tokens_seen": 163577856 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048001003009027083, + "loss": 3.1491, + "theoretical_loss": 4.4677853327209505, + "tokens_seen": 163643392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00048, + "loss": 3.3932, + "theoretical_loss": 4.467554519157452, + "tokens_seen": 163708928 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799899699097292, + "loss": 3.0553, + "theoretical_loss": 4.467323823834824, + "tokens_seen": 163774464 + }, + { + "debugging/Self-BLEU-5": 0.4946583130096161, + "debugging/distinct-1-grams": 0.815790683224271, + "debugging/distinct-2-grams": 0.9751385384195338, + "debugging/entropy-1-grams": 5.884314010514224, + "debugging/entropy-2-grams": 6.564196933680893, + "debugging/length": 580.8, + "debugging/num_segments": 10, + "epoch": 0.05, + "objective/train/docs_used": 101353, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.959970474243164, + "objective/train/theoretical_loss": 4.467093246645215, + "objective/train/tokens_used": 184300000, + "theoretical_loss": 4.467093246645215, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799799398194584, + "loss": 3.3758, + "theoretical_loss": 4.467093246645215, + "tokens_seen": 163840000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047996990972918756, + "loss": 3.0913, + "theoretical_loss": 4.466862787480915, + "tokens_seen": 163905536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047995987963891674, + "loss": 3.201, + "theoretical_loss": 4.466632446234355, + "tokens_seen": 163971072 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047994984954864597, + "loss": 3.1657, + "theoretical_loss": 4.466402222798108, + "tokens_seen": 164036608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799398194583751, + "loss": 3.0971, + "theoretical_loss": 4.466172117064889, + "tokens_seen": 164102144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047992978936810433, + "loss": 3.0109, + "theoretical_loss": 4.465942128927549, + "tokens_seen": 164167680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799197592778335, + "loss": 3.2315, + "theoretical_loss": 4.4657122582790825, + "tokens_seen": 164233216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004799097291875627, + "loss": 3.2653, + "theoretical_loss": 4.465482505012624, + "tokens_seen": 164298752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798996990972919, + "loss": 3.359, + "theoretical_loss": 4.465252869021448, + "tokens_seen": 164364288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047988966900702106, + "loss": 3.0349, + "theoretical_loss": 4.465023350198967, + "tokens_seen": 164429824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047987963891675024, + "loss": 3.4322, + "theoretical_loss": 4.464793948438733, + "tokens_seen": 164495360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798696088264795, + "loss": 3.1553, + "theoretical_loss": 4.464564663634439, + "tokens_seen": 164560896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798595787362086, + "loss": 3.3483, + "theoretical_loss": 4.464335495679914, + "tokens_seen": 164626432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047984954864593784, + "loss": 3.2043, + "theoretical_loss": 4.464106444469129, + "tokens_seen": 164691968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047983951855566696, + "loss": 3.1635, + "theoretical_loss": 4.463877509896189, + "tokens_seen": 164757504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798294884653962, + "loss": 3.298, + "theoretical_loss": 4.463648691855342, + "tokens_seen": 164823040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004798194583751254, + "loss": 3.3382, + "theoretical_loss": 4.46341999024097, + "tokens_seen": 164888576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047980942828485456, + "loss": 3.3223, + "theoretical_loss": 4.463191404947595, + "tokens_seen": 164954112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047979939819458374, + "loss": 3.2243, + "theoretical_loss": 4.462962935869875, + "tokens_seen": 165019648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797893681043129, + "loss": 3.1862, + "theoretical_loss": 4.4627345829026055, + "tokens_seen": 165085184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797793380140421, + "loss": 3.2557, + "theoretical_loss": 4.462506345940719, + "tokens_seen": 165150720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047976930792377134, + "loss": 2.9256, + "theoretical_loss": 4.462278224879286, + "tokens_seen": 165216256 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047975927783350047, + "loss": 2.9158, + "theoretical_loss": 4.462050219613511, + "tokens_seen": 165281792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797492477432297, + "loss": 3.2504, + "theoretical_loss": 4.461822330038737, + "tokens_seen": 165347328 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797392176529589, + "loss": 3.042, + "theoretical_loss": 4.461594556050444, + "tokens_seen": 165412864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 101896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.066077470779419, + "objective/train/theoretical_loss": 4.4613668975442415, + "objective/train/tokens_used": 185938400, + "theoretical_loss": 4.4613668975442415, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047972918756268807, + "loss": 3.1283, + "theoretical_loss": 4.4613668975442415, + "tokens_seen": 165478400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004797191574724173, + "loss": 3.3684, + "theoretical_loss": 4.4611393544158835, + "tokens_seen": 165543936 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047970912738214643, + "loss": 3.3009, + "theoretical_loss": 4.460911926561252, + "tokens_seen": 165609472 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047969909729187566, + "loss": 3.4005, + "theoretical_loss": 4.460684613876367, + "tokens_seen": 165675008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047968906720160484, + "loss": 3.1465, + "theoretical_loss": 4.460457416257386, + "tokens_seen": 165740544 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479679037111334, + "loss": 2.865, + "theoretical_loss": 4.460230333600595, + "tokens_seen": 165806080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796690070210632, + "loss": 3.2027, + "theoretical_loss": 4.46000336580242, + "tokens_seen": 165871616 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796589769307924, + "loss": 3.3585, + "theoretical_loss": 4.459776512759419, + "tokens_seen": 165937152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047964894684052157, + "loss": 3.2687, + "theoretical_loss": 4.459549774368284, + "tokens_seen": 166002688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004796389167502508, + "loss": 3.4614, + "theoretical_loss": 4.45932315052584, + "tokens_seen": 166068224 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047962888665997993, + "loss": 2.9803, + "theoretical_loss": 4.459096641129047, + "tokens_seen": 166133760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047961885656970917, + "loss": 3.0739, + "theoretical_loss": 4.458870246074998, + "tokens_seen": 166199296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047960882647943835, + "loss": 3.285, + "theoretical_loss": 4.458643965260917, + "tokens_seen": 166264832 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047959879638916753, + "loss": 3.1638, + "theoretical_loss": 4.458417798584164, + "tokens_seen": 166330368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795887662988967, + "loss": 2.905, + "theoretical_loss": 4.45819174594223, + "tokens_seen": 166395904 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795787362086259, + "loss": 3.2569, + "theoretical_loss": 4.457965807232739, + "tokens_seen": 166461440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047956870611835507, + "loss": 3.299, + "theoretical_loss": 4.457739982353445, + "tokens_seen": 166526976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795586760280843, + "loss": 2.9863, + "theoretical_loss": 4.457514271202237, + "tokens_seen": 166592512 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047954864593781343, + "loss": 3.2391, + "theoretical_loss": 4.457288673677134, + "tokens_seen": 166658048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047953861584754267, + "loss": 3.3358, + "theoretical_loss": 4.457063189676287, + "tokens_seen": 166723584 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795285857572718, + "loss": 3.2532, + "theoretical_loss": 4.456837819097978, + "tokens_seen": 166789120 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047951855566700103, + "loss": 3.1618, + "theoretical_loss": 4.45661256184062, + "tokens_seen": 166854656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004795085255767302, + "loss": 3.0743, + "theoretical_loss": 4.456387417802757, + "tokens_seen": 166920192 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794984954864594, + "loss": 3.3956, + "theoretical_loss": 4.456162386883065, + "tokens_seen": 166985728 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794884653961886, + "loss": 3.1311, + "theoretical_loss": 4.455937468980348, + "tokens_seen": 167051264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 103097, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.102741003036499, + "objective/train/theoretical_loss": 4.455712663993541, + "objective/train/tokens_used": 187576800, + "theoretical_loss": 4.455712663993541, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047947843530591776, + "loss": 3.2024, + "theoretical_loss": 4.455712663993541, + "tokens_seen": 167116800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047946840521564694, + "loss": 3.4233, + "theoretical_loss": 4.455487971821709, + "tokens_seen": 167182336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047945837512537617, + "loss": 3.1938, + "theoretical_loss": 4.4552633923640474, + "tokens_seen": 167247872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794483450351053, + "loss": 3.1579, + "theoretical_loss": 4.455038925519882, + "tokens_seen": 167313408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047943831494483453, + "loss": 3.2525, + "theoretical_loss": 4.454814571188665, + "tokens_seen": 167378944 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794282848545637, + "loss": 3.0859, + "theoretical_loss": 4.454590329269979, + "tokens_seen": 167444480 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794182547642929, + "loss": 3.4173, + "theoretical_loss": 4.454366199663537, + "tokens_seen": 167510016 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004794082246740221, + "loss": 3.1989, + "theoretical_loss": 4.4541421822691785, + "tokens_seen": 167575552 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047939819458375126, + "loss": 3.2261, + "theoretical_loss": 4.453918276986872, + "tokens_seen": 167641088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047938816449348044, + "loss": 3.3857, + "theoretical_loss": 4.453694483716717, + "tokens_seen": 167706624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793781344032097, + "loss": 3.2033, + "theoretical_loss": 4.453470802358938, + "tokens_seen": 167772160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793681043129388, + "loss": 3.3288, + "theoretical_loss": 4.453247232813885, + "tokens_seen": 167837696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047935807422266804, + "loss": 3.3873, + "theoretical_loss": 4.453023774982041, + "tokens_seen": 167903232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047934804413239716, + "loss": 3.027, + "theoretical_loss": 4.4528004287640135, + "tokens_seen": 167968768 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793380140421264, + "loss": 2.93, + "theoretical_loss": 4.452577194060538, + "tokens_seen": 168034304 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004793279839518556, + "loss": 3.338, + "theoretical_loss": 4.452354070772476, + "tokens_seen": 168099840 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047931795386158476, + "loss": 3.0985, + "theoretical_loss": 4.452131058800816, + "tokens_seen": 168165376 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047930792377131394, + "loss": 3.0258, + "theoretical_loss": 4.451908158046674, + "tokens_seen": 168230912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792978936810431, + "loss": 3.2144, + "theoretical_loss": 4.451685368411292, + "tokens_seen": 168296448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792878635907723, + "loss": 3.1754, + "theoretical_loss": 4.4514626897960365, + "tokens_seen": 168361984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047927783350050154, + "loss": 2.8836, + "theoretical_loss": 4.451240122102401, + "tokens_seen": 168427520 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047926780341023067, + "loss": 3.2178, + "theoretical_loss": 4.451017665232007, + "tokens_seen": 168493056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792577733199599, + "loss": 2.9942, + "theoretical_loss": 4.4507953190865965, + "tokens_seen": 168558592 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792477432296891, + "loss": 3.2664, + "theoretical_loss": 4.450573083568042, + "tokens_seen": 168624128 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047923771313941827, + "loss": 2.9989, + "theoretical_loss": 4.450350958578337, + "tokens_seen": 168689664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 104347, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.790850877761841, + "objective/train/theoretical_loss": 4.4501289440196015, + "objective/train/tokens_used": 189215200, + "theoretical_loss": 4.4501289440196015, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047922768304914745, + "loss": 3.2164, + "theoretical_loss": 4.4501289440196015, + "tokens_seen": 168755200 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047921765295887663, + "loss": 3.3929, + "theoretical_loss": 4.449907039794081, + "tokens_seen": 168820736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004792076228686058, + "loss": 3.2061, + "theoretical_loss": 4.449685245804145, + "tokens_seen": 168886272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047919759277833504, + "loss": 2.9032, + "theoretical_loss": 4.449463561952285, + "tokens_seen": 168951808 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047918756268806417, + "loss": 3.1277, + "theoretical_loss": 4.44924198814112, + "tokens_seen": 169017344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791775325977934, + "loss": 3.2685, + "theoretical_loss": 4.44902052427339, + "tokens_seen": 169082880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047916750250752253, + "loss": 3.0655, + "theoretical_loss": 4.448799170251961, + "tokens_seen": 169148416 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047915747241725177, + "loss": 3.3278, + "theoretical_loss": 4.448577925979821, + "tokens_seen": 169213952 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047914744232698095, + "loss": 3.2436, + "theoretical_loss": 4.44835679136008, + "tokens_seen": 169279488 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047913741223671013, + "loss": 3.2687, + "theoretical_loss": 4.448135766295974, + "tokens_seen": 169345024 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791273821464393, + "loss": 3.1304, + "theoretical_loss": 4.447914850690859, + "tokens_seen": 169410560 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047911735205616855, + "loss": 3.199, + "theoretical_loss": 4.447694044448216, + "tokens_seen": 169476096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004791073219658977, + "loss": 3.2522, + "theoretical_loss": 4.447473347471645, + "tokens_seen": 169541632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790972918756269, + "loss": 3.1969, + "theoretical_loss": 4.447252759664874, + "tokens_seen": 169607168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047908726178535604, + "loss": 3.0039, + "theoretical_loss": 4.4470322809317455, + "tokens_seen": 169672704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047907723169508527, + "loss": 3.5408, + "theoretical_loss": 4.44681191117623, + "tokens_seen": 169738240 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047906720160481445, + "loss": 3.2186, + "theoretical_loss": 4.446591650302416, + "tokens_seen": 169803776 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047905717151454363, + "loss": 2.9781, + "theoretical_loss": 4.446371498214516, + "tokens_seen": 169869312 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790471414242728, + "loss": 3.4193, + "theoretical_loss": 4.446151454816862, + "tokens_seen": 169934848 + }, + { + "epoch": 0.05, + "learning_rate": 0.000479037111334002, + "loss": 3.0771, + "theoretical_loss": 4.445931520013906, + "tokens_seen": 170000384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790270812437312, + "loss": 3.2464, + "theoretical_loss": 4.445711693710223, + "tokens_seen": 170065920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004790170511534604, + "loss": 3.2014, + "theoretical_loss": 4.4454919758105085, + "tokens_seen": 170131456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047900702106318954, + "loss": 3.3265, + "theoretical_loss": 4.445272366219575, + "tokens_seen": 170196992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789969909729188, + "loss": 3.073, + "theoretical_loss": 4.44505286484236, + "tokens_seen": 170262528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789869608826479, + "loss": 2.936, + "theoretical_loss": 4.4448334715839195, + "tokens_seen": 170328064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 105037, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8210434913635254, + "objective/train/theoretical_loss": 4.444614186349425, + "objective/train/tokens_used": 190853600, + "theoretical_loss": 4.444614186349425, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047897693079237714, + "loss": 2.9571, + "theoretical_loss": 4.444614186349425, + "tokens_seen": 170393600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047896690070210637, + "loss": 2.9778, + "theoretical_loss": 4.444395009044175, + "tokens_seen": 170459136 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789568706118355, + "loss": 3.2967, + "theoretical_loss": 4.444175939573581, + "tokens_seen": 170524672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047894684052156473, + "loss": 2.8109, + "theoretical_loss": 4.443956977843177, + "tokens_seen": 170590208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789368104312939, + "loss": 3.4383, + "theoretical_loss": 4.4437381237586155, + "tokens_seen": 170655744 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789267803410231, + "loss": 3.1635, + "theoretical_loss": 4.443519377225666, + "tokens_seen": 170721280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004789167502507523, + "loss": 3.1526, + "theoretical_loss": 4.443300738150221, + "tokens_seen": 170786816 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047890672016048146, + "loss": 3.3384, + "theoretical_loss": 4.4430822064382856, + "tokens_seen": 170852352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047889669007021064, + "loss": 3.4135, + "theoretical_loss": 4.442863781995989, + "tokens_seen": 170917888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788866599799399, + "loss": 3.0138, + "theoretical_loss": 4.442645464729573, + "tokens_seen": 170983424 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478876629889669, + "loss": 3.2596, + "theoretical_loss": 4.4424272545454, + "tokens_seen": 171048960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047886659979939824, + "loss": 3.2555, + "theoretical_loss": 4.442209151349951, + "tokens_seen": 171114496 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047885656970912736, + "loss": 3.3042, + "theoretical_loss": 4.4419911550498234, + "tokens_seen": 171180032 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788465396188566, + "loss": 3.2061, + "theoretical_loss": 4.441773265551731, + "tokens_seen": 171245568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788365095285858, + "loss": 3.0708, + "theoretical_loss": 4.441555482762507, + "tokens_seen": 171311104 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047882647943831496, + "loss": 3.2816, + "theoretical_loss": 4.441337806589097, + "tokens_seen": 171376640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047881644934804414, + "loss": 3.1222, + "theoretical_loss": 4.441120236938568, + "tokens_seen": 171442176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004788064192577733, + "loss": 3.263, + "theoretical_loss": 4.440902773718101, + "tokens_seen": 171507712 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787963891675025, + "loss": 2.9574, + "theoretical_loss": 4.440685416834994, + "tokens_seen": 171573248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047878635907723174, + "loss": 3.4099, + "theoretical_loss": 4.440468166196662, + "tokens_seen": 171638784 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047877632898696087, + "loss": 3.2833, + "theoretical_loss": 4.440251021710634, + "tokens_seen": 171704320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787662988966901, + "loss": 3.2847, + "theoretical_loss": 4.440033983284556, + "tokens_seen": 171769856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004787562688064193, + "loss": 3.3134, + "theoretical_loss": 4.43981705082619, + "tokens_seen": 171835392 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047874623871614847, + "loss": 3.1022, + "theoretical_loss": 4.439600224243412, + "tokens_seen": 171900928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047873620862587765, + "loss": 3.1256, + "theoretical_loss": 4.439383503444214, + "tokens_seen": 171966464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 106163, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.448519468307495, + "objective/train/theoretical_loss": 4.439166888336702, + "objective/train/tokens_used": 192492000, + "theoretical_loss": 4.439166888336702, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047872617853560683, + "loss": 3.3453, + "theoretical_loss": 4.439166888336702, + "tokens_seen": 172032000 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478716148445336, + "loss": 3.0654, + "theoretical_loss": 4.438950378829098, + "tokens_seen": 172097536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047870611835506524, + "loss": 3.3586, + "theoretical_loss": 4.43873397482974, + "tokens_seen": 172163072 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047869608826479437, + "loss": 3.1219, + "theoretical_loss": 4.438517676247075, + "tokens_seen": 172228608 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786860581745236, + "loss": 3.0914, + "theoretical_loss": 4.4383014829896705, + "tokens_seen": 172294144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047867602808425273, + "loss": 3.1461, + "theoretical_loss": 4.4380853949662065, + "tokens_seen": 172359680 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047866599799398197, + "loss": 3.0581, + "theoretical_loss": 4.4378694120854725, + "tokens_seen": 172425216 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047865596790371115, + "loss": 3.0474, + "theoretical_loss": 4.437653534256377, + "tokens_seen": 172490752 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047864593781344033, + "loss": 3.207, + "theoretical_loss": 4.437437761387939, + "tokens_seen": 172556288 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786359077231695, + "loss": 3.1347, + "theoretical_loss": 4.437222093389293, + "tokens_seen": 172621824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047862587763289875, + "loss": 3.0996, + "theoretical_loss": 4.437006530169684, + "tokens_seen": 172687360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786158475426279, + "loss": 3.255, + "theoretical_loss": 4.436791071638472, + "tokens_seen": 172752896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004786058174523571, + "loss": 3.3615, + "theoretical_loss": 4.43657571770513, + "tokens_seen": 172818432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047859578736208624, + "loss": 3.3575, + "theoretical_loss": 4.436360468279242, + "tokens_seen": 172883968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047858575727181547, + "loss": 3.1795, + "theoretical_loss": 4.4361453232705035, + "tokens_seen": 172949504 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047857572718154465, + "loss": 3.2246, + "theoretical_loss": 4.435930282588725, + "tokens_seen": 173015040 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047856569709127383, + "loss": 3.1589, + "theoretical_loss": 4.435715346143828, + "tokens_seen": 173080576 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478555667001003, + "loss": 3.1079, + "theoretical_loss": 4.435500513845845, + "tokens_seen": 173146112 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785456369107322, + "loss": 3.0886, + "theoretical_loss": 4.435285785604922, + "tokens_seen": 173211648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785356068204614, + "loss": 3.3142, + "theoretical_loss": 4.435071161331313, + "tokens_seen": 173277184 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004785255767301906, + "loss": 3.3968, + "theoretical_loss": 4.434856640935388, + "tokens_seen": 173342720 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047851554663991974, + "loss": 3.1791, + "theoretical_loss": 4.434642224327625, + "tokens_seen": 173408256 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478505516549649, + "loss": 3.1941, + "theoretical_loss": 4.434427911418613, + "tokens_seen": 173473792 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784954864593781, + "loss": 3.1568, + "theoretical_loss": 4.434213702119052, + "tokens_seen": 173539328 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047848545636910734, + "loss": 3.3522, + "theoretical_loss": 4.433999596339755, + "tokens_seen": 173604864 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 106755, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8909780979156494, + "objective/train/theoretical_loss": 4.433785593991642, + "objective/train/tokens_used": 194130400, + "theoretical_loss": 4.433785593991642, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784754262788365, + "loss": 3.2649, + "theoretical_loss": 4.433785593991642, + "tokens_seen": 173670400 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784653961885657, + "loss": 3.2448, + "theoretical_loss": 4.433571694985744, + "tokens_seen": 173735936 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784553660982949, + "loss": 3.2039, + "theoretical_loss": 4.433357899233204, + "tokens_seen": 173801472 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784453360080241, + "loss": 3.0269, + "theoretical_loss": 4.433144206645274, + "tokens_seen": 173867008 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047843530591775324, + "loss": 3.0917, + "theoretical_loss": 4.432930617133313, + "tokens_seen": 173932544 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784252758274825, + "loss": 3.1783, + "theoretical_loss": 4.432717130608793, + "tokens_seen": 173998080 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004784152457372116, + "loss": 3.1232, + "theoretical_loss": 4.432503746983295, + "tokens_seen": 174063616 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047840521564694084, + "loss": 3.2631, + "theoretical_loss": 4.432290466168507, + "tokens_seen": 174129152 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047839518555667, + "loss": 3.2136, + "theoretical_loss": 4.432077288076228, + "tokens_seen": 174194688 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783851554663992, + "loss": 3.2796, + "theoretical_loss": 4.431864212618365, + "tokens_seen": 174260224 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783751253761284, + "loss": 3.4694, + "theoretical_loss": 4.431651239706934, + "tokens_seen": 174325760 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047836509528585756, + "loss": 3.0788, + "theoretical_loss": 4.431438369254058, + "tokens_seen": 174391296 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047835506519558675, + "loss": 3.0872, + "theoretical_loss": 4.43122560117197, + "tokens_seen": 174456832 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478345035105316, + "loss": 2.9119, + "theoretical_loss": 4.431012935373012, + "tokens_seen": 174522368 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783350050150451, + "loss": 3.2154, + "theoretical_loss": 4.430800371769631, + "tokens_seen": 174587904 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047832497492477434, + "loss": 3.2817, + "theoretical_loss": 4.430587910274383, + "tokens_seen": 174653440 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047831494483450347, + "loss": 3.1007, + "theoretical_loss": 4.430375550799933, + "tokens_seen": 174718976 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004783049147442327, + "loss": 3.3202, + "theoretical_loss": 4.430163293259053, + "tokens_seen": 174784512 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782948846539619, + "loss": 3.2775, + "theoretical_loss": 4.42995113756462, + "tokens_seen": 174850048 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047828485456369107, + "loss": 3.4485, + "theoretical_loss": 4.42973908362962, + "tokens_seen": 174915584 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047827482447342025, + "loss": 3.1023, + "theoretical_loss": 4.429527131367147, + "tokens_seen": 174981120 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782647943831495, + "loss": 3.0822, + "theoretical_loss": 4.429315280690398, + "tokens_seen": 175046656 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782547642928786, + "loss": 3.2901, + "theoretical_loss": 4.4291035315126805, + "tokens_seen": 175112192 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047824473420260785, + "loss": 3.2442, + "theoretical_loss": 4.428891883747406, + "tokens_seen": 175177728 + }, + { + "epoch": 0.05, + "learning_rate": 0.000478234704112337, + "loss": 3.166, + "theoretical_loss": 4.428680337308094, + "tokens_seen": 175243264 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 107724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6482720375061035, + "objective/train/theoretical_loss": 4.428468892108368, + "objective/train/tokens_used": 195768800, + "theoretical_loss": 4.428468892108368, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004782246740220662, + "loss": 3.3804, + "theoretical_loss": 4.428468892108368, + "tokens_seen": 175308800 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047821464393179544, + "loss": 3.4307, + "theoretical_loss": 4.428257548061959, + "tokens_seen": 175374336 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047820461384152457, + "loss": 3.2556, + "theoretical_loss": 4.428046305082704, + "tokens_seen": 175439872 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781945837512538, + "loss": 3.3203, + "theoretical_loss": 4.427835163084543, + "tokens_seen": 175505408 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047818455366098293, + "loss": 3.1723, + "theoretical_loss": 4.427624121981523, + "tokens_seen": 175570944 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047817452357071217, + "loss": 3.1751, + "theoretical_loss": 4.427413181687799, + "tokens_seen": 175636480 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047816449348044135, + "loss": 2.9857, + "theoretical_loss": 4.427202342117624, + "tokens_seen": 175702016 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047815446339017053, + "loss": 3.1727, + "theoretical_loss": 4.426991603185365, + "tokens_seen": 175767552 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781444332998997, + "loss": 3.121, + "theoretical_loss": 4.426780964805486, + "tokens_seen": 175833088 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047813440320962895, + "loss": 3.3494, + "theoretical_loss": 4.4265704268925585, + "tokens_seen": 175898624 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781243731193581, + "loss": 3.313, + "theoretical_loss": 4.426359989361259, + "tokens_seen": 175964160 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004781143430290873, + "loss": 3.3073, + "theoretical_loss": 4.426149652126369, + "tokens_seen": 176029696 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047810431293881644, + "loss": 3.0604, + "theoretical_loss": 4.42593941510277, + "tokens_seen": 176095232 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047809428284854567, + "loss": 2.7989, + "theoretical_loss": 4.42572927820545, + "tokens_seen": 176160768 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047808425275827485, + "loss": 3.2073, + "theoretical_loss": 4.4255192413495035, + "tokens_seen": 176226304 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047807422266800403, + "loss": 3.0824, + "theoretical_loss": 4.425309304450122, + "tokens_seen": 176291840 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780641925777332, + "loss": 3.1597, + "theoretical_loss": 4.4250994674226085, + "tokens_seen": 176357376 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780541624874624, + "loss": 3.1731, + "theoretical_loss": 4.424889730182361, + "tokens_seen": 176422912 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780441323971916, + "loss": 3.3307, + "theoretical_loss": 4.424680092644886, + "tokens_seen": 176488448 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780341023069208, + "loss": 3.0847, + "theoretical_loss": 4.42447055472579, + "tokens_seen": 176553984 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047802407221664994, + "loss": 3.3959, + "theoretical_loss": 4.4242611163407854, + "tokens_seen": 176619520 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780140421263792, + "loss": 3.2021, + "theoretical_loss": 4.424051777405684, + "tokens_seen": 176685056 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004780040120361083, + "loss": 3.0791, + "theoretical_loss": 4.423842537836402, + "tokens_seen": 176750592 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047799398194583754, + "loss": 3.112, + "theoretical_loss": 4.423633397548956, + "tokens_seen": 176816128 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779839518555667, + "loss": 3.0609, + "theoretical_loss": 4.423424356459467, + "tokens_seen": 176881664 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 108296, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4872632026672363, + "objective/train/theoretical_loss": 4.423215414484155, + "objective/train/tokens_used": 197407200, + "theoretical_loss": 4.423215414484155, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779739217652959, + "loss": 3.3284, + "theoretical_loss": 4.423215414484155, + "tokens_seen": 176947200 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779638916750251, + "loss": 2.839, + "theoretical_loss": 4.423006571539346, + "tokens_seen": 177012736 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779538615847543, + "loss": 3.1684, + "theoretical_loss": 4.422797827541462, + "tokens_seen": 177078272 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047794383149448344, + "loss": 3.1424, + "theoretical_loss": 4.422589182407032, + "tokens_seen": 177143808 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779338014042127, + "loss": 3.2199, + "theoretical_loss": 4.422380636052681, + "tokens_seen": 177209344 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779237713139418, + "loss": 2.9687, + "theoretical_loss": 4.422172188395141, + "tokens_seen": 177274880 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047791374122367104, + "loss": 3.0297, + "theoretical_loss": 4.421963839351239, + "tokens_seen": 177340416 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004779037111334002, + "loss": 3.1726, + "theoretical_loss": 4.421755588837907, + "tokens_seen": 177405952 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778936810431294, + "loss": 3.3507, + "theoretical_loss": 4.421547436772176, + "tokens_seen": 177471488 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778836509528586, + "loss": 3.2595, + "theoretical_loss": 4.421339383071178, + "tokens_seen": 177537024 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047787362086258776, + "loss": 3.3193, + "theoretical_loss": 4.421131427652145, + "tokens_seen": 177602560 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047786359077231695, + "loss": 3.2873, + "theoretical_loss": 4.420923570432407, + "tokens_seen": 177668096 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778535606820462, + "loss": 3.1947, + "theoretical_loss": 4.420715811329398, + "tokens_seen": 177733632 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778435305917753, + "loss": 3.2383, + "theoretical_loss": 4.420508150260651, + "tokens_seen": 177799168 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047783350050150454, + "loss": 3.1387, + "theoretical_loss": 4.420300587143795, + "tokens_seen": 177864704 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047782347041123367, + "loss": 2.8977, + "theoretical_loss": 4.420093121896563, + "tokens_seen": 177930240 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778134403209629, + "loss": 3.1988, + "theoretical_loss": 4.419885754436784, + "tokens_seen": 177995776 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004778034102306921, + "loss": 3.0306, + "theoretical_loss": 4.4196784846823896, + "tokens_seen": 178061312 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047779338014042127, + "loss": 3.1273, + "theoretical_loss": 4.419471312551408, + "tokens_seen": 178126848 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047778335005015045, + "loss": 2.9744, + "theoretical_loss": 4.419264237961966, + "tokens_seen": 178192384 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777733199598797, + "loss": 3.0912, + "theoretical_loss": 4.41905726083229, + "tokens_seen": 178257920 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777632898696088, + "loss": 3.2138, + "theoretical_loss": 4.418850381080706, + "tokens_seen": 178323456 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047775325977933805, + "loss": 3.2271, + "theoretical_loss": 4.4186435986256365, + "tokens_seen": 178388992 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777432296890672, + "loss": 3.0616, + "theoretical_loss": 4.418436913385604, + "tokens_seen": 178454528 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777331995987964, + "loss": 3.2644, + "theoretical_loss": 4.418230325279228, + "tokens_seen": 178520064 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 109542, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0579349994659424, + "objective/train/theoretical_loss": 4.418023834225226, + "objective/train/tokens_used": 199045600, + "theoretical_loss": 4.418023834225226, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004777231695085256, + "loss": 3.1012, + "theoretical_loss": 4.418023834225226, + "tokens_seen": 178585600 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047771313941825477, + "loss": 3.1625, + "theoretical_loss": 4.417817440142414, + "tokens_seen": 178651136 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047770310932798395, + "loss": 3.0349, + "theoretical_loss": 4.417611142949705, + "tokens_seen": 178716672 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047769307923771313, + "loss": 3.2227, + "theoretical_loss": 4.417404942566111, + "tokens_seen": 178782208 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776830491474423, + "loss": 3.0844, + "theoretical_loss": 4.417198838910737, + "tokens_seen": 178847744 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047767301905717155, + "loss": 3.3573, + "theoretical_loss": 4.4169928319027925, + "tokens_seen": 178913280 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776629889669007, + "loss": 3.297, + "theoretical_loss": 4.416786921461577, + "tokens_seen": 178978816 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776529588766299, + "loss": 3.2404, + "theoretical_loss": 4.41658110750649, + "tokens_seen": 179044352 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047764292878635904, + "loss": 3.2353, + "theoretical_loss": 4.416375389957029, + "tokens_seen": 179109888 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776328986960883, + "loss": 3.2372, + "theoretical_loss": 4.416169768732786, + "tokens_seen": 179175424 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047762286860581746, + "loss": 3.1656, + "theoretical_loss": 4.415964243753448, + "tokens_seen": 179240960 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047761283851554664, + "loss": 2.8394, + "theoretical_loss": 4.415758814938803, + "tokens_seen": 179306496 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004776028084252758, + "loss": 3.3786, + "theoretical_loss": 4.415553482208733, + "tokens_seen": 179372032 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047759277833500505, + "loss": 3.2626, + "theoretical_loss": 4.415348245483212, + "tokens_seen": 179437568 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775827482447342, + "loss": 3.2584, + "theoretical_loss": 4.415143104682317, + "tokens_seen": 179503104 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775727181544634, + "loss": 3.2606, + "theoretical_loss": 4.4149380597262144, + "tokens_seen": 179568640 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047756268806419254, + "loss": 3.3959, + "theoretical_loss": 4.41473311053517, + "tokens_seen": 179634176 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775526579739218, + "loss": 3.1822, + "theoretical_loss": 4.414528257029543, + "tokens_seen": 179699712 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047754262788365096, + "loss": 3.3075, + "theoretical_loss": 4.414323499129791, + "tokens_seen": 179765248 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047753259779338014, + "loss": 3.1764, + "theoretical_loss": 4.414118836756462, + "tokens_seen": 179830784 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775225677031093, + "loss": 3.2029, + "theoretical_loss": 4.4139142698302, + "tokens_seen": 179896320 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775125376128385, + "loss": 3.0117, + "theoretical_loss": 4.413709798271748, + "tokens_seen": 179961856 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004775025075225677, + "loss": 3.0247, + "theoretical_loss": 4.41350542200194, + "tokens_seen": 180027392 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774924774322969, + "loss": 3.2042, + "theoretical_loss": 4.413301140941703, + "tokens_seen": 180092928 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047748244734202605, + "loss": 3.1478, + "theoretical_loss": 4.413096955012063, + "tokens_seen": 180158464 + }, + { + "epoch": 0.05, + "objective/train/docs_used": 110066, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8472869396209717, + "objective/train/theoretical_loss": 4.412892864134137, + "objective/train/tokens_used": 200684000, + "theoretical_loss": 4.412892864134137, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774724172517553, + "loss": 3.0943, + "theoretical_loss": 4.412892864134137, + "tokens_seen": 180224000 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774623871614845, + "loss": 2.8881, + "theoretical_loss": 4.412688868229136, + "tokens_seen": 180289536 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047745235707121364, + "loss": 3.3796, + "theoretical_loss": 4.412484967218368, + "tokens_seen": 180355072 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774423269809429, + "loss": 3.234, + "theoretical_loss": 4.4122811610232295, + "tokens_seen": 180420608 + }, + { + "epoch": 0.05, + "learning_rate": 0.000477432296890672, + "loss": 3.1805, + "theoretical_loss": 4.412077449565216, + "tokens_seen": 180486144 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047742226680040124, + "loss": 3.4462, + "theoretical_loss": 4.411873832765913, + "tokens_seen": 180551680 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774122367101304, + "loss": 3.1975, + "theoretical_loss": 4.4116703105470005, + "tokens_seen": 180617216 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004774022066198596, + "loss": 3.1883, + "theoretical_loss": 4.411466882830252, + "tokens_seen": 180682752 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773921765295888, + "loss": 3.1599, + "theoretical_loss": 4.411263549537534, + "tokens_seen": 180748288 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047738214643931797, + "loss": 3.1214, + "theoretical_loss": 4.411060310590805, + "tokens_seen": 180813824 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047737211634904715, + "loss": 3.0375, + "theoretical_loss": 4.410857165912116, + "tokens_seen": 180879360 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773620862587764, + "loss": 3.0394, + "theoretical_loss": 4.410654115423613, + "tokens_seen": 180944896 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773520561685055, + "loss": 3.0477, + "theoretical_loss": 4.410451159047533, + "tokens_seen": 181010432 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047734202607823474, + "loss": 3.2956, + "theoretical_loss": 4.410248296706204, + "tokens_seen": 181075968 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047733199598796387, + "loss": 3.1011, + "theoretical_loss": 4.410045528322048, + "tokens_seen": 181141504 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773219658976931, + "loss": 3.1455, + "theoretical_loss": 4.409842853817577, + "tokens_seen": 181207040 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004773119358074223, + "loss": 3.1878, + "theoretical_loss": 4.409640273115399, + "tokens_seen": 181272576 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047730190571715147, + "loss": 2.8744, + "theoretical_loss": 4.409437786138211, + "tokens_seen": 181338112 + }, + { + "epoch": 0.05, + "learning_rate": 0.00047729187562688065, + "loss": 3.1718, + "theoretical_loss": 4.4092353928087995, + "tokens_seen": 181403648 + }, + { + "epoch": 0.05, + "learning_rate": 0.0004772818455366099, + "loss": 3.1126, + "theoretical_loss": 4.409033093050046, + "tokens_seen": 181469184 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477271815446339, + "loss": 3.4133, + "theoretical_loss": 4.408830886784922, + "tokens_seen": 181534720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047726178535606825, + "loss": 3.2217, + "theoretical_loss": 4.40862877393649, + "tokens_seen": 181600256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772517552657974, + "loss": 3.1425, + "theoretical_loss": 4.408426754427904, + "tokens_seen": 181665792 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772417251755266, + "loss": 3.009, + "theoretical_loss": 4.408224828182409, + "tokens_seen": 181731328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004772316950852558, + "loss": 3.0163, + "theoretical_loss": 4.40802299512334, + "tokens_seen": 181796864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 111369, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.439749002456665, + "objective/train/theoretical_loss": 4.407821255174123, + "objective/train/tokens_used": 202322400, + "theoretical_loss": 4.407821255174123, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047722166499498497, + "loss": 3.1176, + "theoretical_loss": 4.407821255174123, + "tokens_seen": 181862400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047721163490471415, + "loss": 3.3191, + "theoretical_loss": 4.407619608258274, + "tokens_seen": 181927936 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047720160481444333, + "loss": 3.1351, + "theoretical_loss": 4.407418054299402, + "tokens_seen": 181993472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771915747241725, + "loss": 3.2347, + "theoretical_loss": 4.407216593221202, + "tokens_seen": 182059008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047718154463390175, + "loss": 3.182, + "theoretical_loss": 4.407015224947463, + "tokens_seen": 182124544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771715145436309, + "loss": 3.3172, + "theoretical_loss": 4.40681394940206, + "tokens_seen": 182190080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771614844533601, + "loss": 3.3428, + "theoretical_loss": 4.406612766508961, + "tokens_seen": 182255616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047715145436308924, + "loss": 3.1138, + "theoretical_loss": 4.4064116761922225, + "tokens_seen": 182321152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004771414242728185, + "loss": 2.9224, + "theoretical_loss": 4.406210678375992, + "tokens_seen": 182386688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047713139418254766, + "loss": 3.0207, + "theoretical_loss": 4.406009772984502, + "tokens_seen": 182452224 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047712136409227684, + "loss": 2.9416, + "theoretical_loss": 4.40580895994208, + "tokens_seen": 182517760 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477111334002006, + "loss": 3.1591, + "theoretical_loss": 4.405608239173138, + "tokens_seen": 182583296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047710130391173525, + "loss": 3.142, + "theoretical_loss": 4.405407610602181, + "tokens_seen": 182648832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770912738214644, + "loss": 3.2257, + "theoretical_loss": 4.405207074153797, + "tokens_seen": 182714368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770812437311936, + "loss": 3.0585, + "theoretical_loss": 4.40500662975267, + "tokens_seen": 182779904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047707121364092274, + "loss": 3.0379, + "theoretical_loss": 4.4048062773235666, + "tokens_seen": 182845440 + }, + { + "epoch": 0.06, + "learning_rate": 0.000477061183550652, + "loss": 3.018, + "theoretical_loss": 4.404606016791345, + "tokens_seen": 182910976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047705115346038116, + "loss": 3.1405, + "theoretical_loss": 4.404405848080951, + "tokens_seen": 182976512 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047704112337011034, + "loss": 2.9844, + "theoretical_loss": 4.404205771117417, + "tokens_seen": 183042048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770310932798395, + "loss": 3.0418, + "theoretical_loss": 4.404005785825866, + "tokens_seen": 183107584 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770210631895687, + "loss": 3.2314, + "theoretical_loss": 4.403805892131508, + "tokens_seen": 183173120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770110330992979, + "loss": 3.2265, + "theoretical_loss": 4.40360608995964, + "tokens_seen": 183238656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004770010030090271, + "loss": 3.0284, + "theoretical_loss": 4.403406379235646, + "tokens_seen": 183304192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047699097291875625, + "loss": 3.0799, + "theoretical_loss": 4.403206759884999, + "tokens_seen": 183369728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769809428284855, + "loss": 3.0723, + "theoretical_loss": 4.403007231833261, + "tokens_seen": 183435264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 111925, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.625623941421509, + "objective/train/theoretical_loss": 4.402807795006074, + "objective/train/tokens_used": 203960800, + "theoretical_loss": 4.402807795006074, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047697091273821466, + "loss": 3.2865, + "theoretical_loss": 4.402807795006074, + "tokens_seen": 183500800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047696088264794384, + "loss": 2.8014, + "theoretical_loss": 4.402608449329178, + "tokens_seen": 183566336 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476950852557673, + "loss": 3.007, + "theoretical_loss": 4.40240919472839, + "tokens_seen": 183631872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769408224674022, + "loss": 3.0607, + "theoretical_loss": 4.40221003112962, + "tokens_seen": 183697408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769307923771314, + "loss": 3.393, + "theoretical_loss": 4.4020109584588605, + "tokens_seen": 183762944 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004769207622868606, + "loss": 3.3108, + "theoretical_loss": 4.401811976642195, + "tokens_seen": 183828480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047691073219658975, + "loss": 2.9578, + "theoretical_loss": 4.4016130856057885, + "tokens_seen": 183894016 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476900702106319, + "loss": 3.2152, + "theoretical_loss": 4.401414285275898, + "tokens_seen": 183959552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768906720160481, + "loss": 3.2252, + "theoretical_loss": 4.401215575578861, + "tokens_seen": 184025088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047688064192577735, + "loss": 3.0185, + "theoretical_loss": 4.4010169564411035, + "tokens_seen": 184090624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047687061183550653, + "loss": 3.0891, + "theoretical_loss": 4.4008184277891385, + "tokens_seen": 184156160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768605817452357, + "loss": 3.1172, + "theoretical_loss": 4.400619989549563, + "tokens_seen": 184221696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768505516549649, + "loss": 3.0661, + "theoretical_loss": 4.400421641649061, + "tokens_seen": 184287232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047684052156469407, + "loss": 2.9647, + "theoretical_loss": 4.400223384014399, + "tokens_seen": 184352768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047683049147442325, + "loss": 3.0501, + "theoretical_loss": 4.400025216572435, + "tokens_seen": 184418304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768204613841525, + "loss": 2.9332, + "theoretical_loss": 4.399827139250106, + "tokens_seen": 184483840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004768104312938816, + "loss": 3.2431, + "theoretical_loss": 4.399629151974436, + "tokens_seen": 184549376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047680040120361085, + "loss": 2.9531, + "theoretical_loss": 4.399431254672535, + "tokens_seen": 184614912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047679037111334003, + "loss": 3.1576, + "theoretical_loss": 4.399233447271598, + "tokens_seen": 184680448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767803410230692, + "loss": 3.1937, + "theoretical_loss": 4.399035729698903, + "tokens_seen": 184745984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767703109327984, + "loss": 3.0762, + "theoretical_loss": 4.398838101881815, + "tokens_seen": 184811520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767602808425276, + "loss": 3.1412, + "theoretical_loss": 4.39864056374778, + "tokens_seen": 184877056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047675025075225676, + "loss": 3.2723, + "theoretical_loss": 4.398443115224333, + "tokens_seen": 184942592 + }, + { + "epoch": 0.06, + "learning_rate": 0.000476740220661986, + "loss": 3.3477, + "theoretical_loss": 4.398245756239088, + "tokens_seen": 185008128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767301905717151, + "loss": 2.7878, + "theoretical_loss": 4.398048486719748, + "tokens_seen": 185073664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 113096, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7102208137512207, + "objective/train/theoretical_loss": 4.397851306594097, + "objective/train/tokens_used": 205599200, + "theoretical_loss": 4.397851306594097, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047672016048144435, + "loss": 2.8609, + "theoretical_loss": 4.397851306594097, + "tokens_seen": 185139200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047671013039117353, + "loss": 3.0009, + "theoretical_loss": 4.397654215790003, + "tokens_seen": 185204736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004767001003009027, + "loss": 3.029, + "theoretical_loss": 4.397457214235419, + "tokens_seen": 185270272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047669007021063195, + "loss": 3.292, + "theoretical_loss": 4.397260301858379, + "tokens_seen": 185335808 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766800401203611, + "loss": 3.1928, + "theoretical_loss": 4.397063478587005, + "tokens_seen": 185401344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766700100300903, + "loss": 3.3045, + "theoretical_loss": 4.396866744349498, + "tokens_seen": 185466880 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047665997993981944, + "loss": 3.1277, + "theoretical_loss": 4.396670099074143, + "tokens_seen": 185532416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766499498495487, + "loss": 3.0829, + "theoretical_loss": 4.39647354268931, + "tokens_seen": 185597952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047663991975927786, + "loss": 3.2087, + "theoretical_loss": 4.39627707512345, + "tokens_seen": 185663488 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047662988966900704, + "loss": 2.8462, + "theoretical_loss": 4.3960806963050985, + "tokens_seen": 185729024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004766198595787362, + "loss": 3.0272, + "theoretical_loss": 4.395884406162873, + "tokens_seen": 185794560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047660982948846545, + "loss": 3.2038, + "theoretical_loss": 4.395688204625471, + "tokens_seen": 185860096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765997993981946, + "loss": 2.8081, + "theoretical_loss": 4.395492091621677, + "tokens_seen": 185925632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765897693079238, + "loss": 3.3035, + "theoretical_loss": 4.395296067080356, + "tokens_seen": 185991168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047657973921765294, + "loss": 3.1633, + "theoretical_loss": 4.3951001309304525, + "tokens_seen": 186056704 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765697091273822, + "loss": 3.145, + "theoretical_loss": 4.394904283100997, + "tokens_seen": 186122240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047655967903711136, + "loss": 3.2733, + "theoretical_loss": 4.394708523521101, + "tokens_seen": 186187776 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047654964894684054, + "loss": 3.1495, + "theoretical_loss": 4.394512852119956, + "tokens_seen": 186253312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765396188565697, + "loss": 3.1897, + "theoretical_loss": 4.3943172688268355, + "tokens_seen": 186318848 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765295887662989, + "loss": 3.0105, + "theoretical_loss": 4.394121773571097, + "tokens_seen": 186384384 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765195586760281, + "loss": 3.1493, + "theoretical_loss": 4.3939263662821775, + "tokens_seen": 186449920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004765095285857573, + "loss": 2.9624, + "theoretical_loss": 4.393731046889595, + "tokens_seen": 186515456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047649949849548645, + "loss": 3.3138, + "theoretical_loss": 4.3935358153229505, + "tokens_seen": 186580992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764894684052157, + "loss": 3.1593, + "theoretical_loss": 4.3933406715119245, + "tokens_seen": 186646528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047647943831494486, + "loss": 3.0466, + "theoretical_loss": 4.393145615386279, + "tokens_seen": 186712064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 113525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.944962978363037, + "objective/train/theoretical_loss": 4.392950646875857, + "objective/train/tokens_used": 207237600, + "theoretical_loss": 4.392950646875857, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047646940822467404, + "loss": 3.0963, + "theoretical_loss": 4.392950646875857, + "tokens_seen": 186777600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764593781344032, + "loss": 3.0612, + "theoretical_loss": 4.392755765910581, + "tokens_seen": 186843136 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764493480441324, + "loss": 3.1482, + "theoretical_loss": 4.3925609724204575, + "tokens_seen": 186908672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764393179538616, + "loss": 3.2602, + "theoretical_loss": 4.392366266335569, + "tokens_seen": 186974208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764292878635908, + "loss": 3.1856, + "theoretical_loss": 4.392171647586081, + "tokens_seen": 187039744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047641925777331995, + "loss": 3.0132, + "theoretical_loss": 4.39197711610224, + "tokens_seen": 187105280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004764092276830492, + "loss": 3.152, + "theoretical_loss": 4.391782671814369, + "tokens_seen": 187170816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763991975927783, + "loss": 3.2149, + "theoretical_loss": 4.391588314652875, + "tokens_seen": 187236352 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047638916750250755, + "loss": 3.2574, + "theoretical_loss": 4.3913940445482424, + "tokens_seen": 187301888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047637913741223673, + "loss": 3.1899, + "theoretical_loss": 4.391199861431037, + "tokens_seen": 187367424 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763691073219659, + "loss": 3.0975, + "theoretical_loss": 4.3910057652319034, + "tokens_seen": 187432960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763590772316951, + "loss": 3.2897, + "theoretical_loss": 4.390811755881566, + "tokens_seen": 187498496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047634904714142427, + "loss": 3.3427, + "theoretical_loss": 4.390617833310827, + "tokens_seen": 187564032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047633901705115345, + "loss": 3.181, + "theoretical_loss": 4.390423997450571, + "tokens_seen": 187629568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763289869608827, + "loss": 3.1201, + "theoretical_loss": 4.3902302482317594, + "tokens_seen": 187695104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004763189568706118, + "loss": 3.12, + "theoretical_loss": 4.390036585585432, + "tokens_seen": 187760640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047630892678034105, + "loss": 2.9343, + "theoretical_loss": 4.3898430094427106, + "tokens_seen": 187826176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047629889669007023, + "loss": 3.0305, + "theoretical_loss": 4.389649519734793, + "tokens_seen": 187891712 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762888665997994, + "loss": 3.0496, + "theoretical_loss": 4.3894561163929575, + "tokens_seen": 187957248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762788365095286, + "loss": 3.2563, + "theoretical_loss": 4.389262799348559, + "tokens_seen": 188022784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762688064192578, + "loss": 3.2352, + "theoretical_loss": 4.389069568533033, + "tokens_seen": 188088320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047625877632898696, + "loss": 3.1003, + "theoretical_loss": 4.388876423877893, + "tokens_seen": 188153856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762487462387162, + "loss": 3.1905, + "theoretical_loss": 4.388683365314728, + "tokens_seen": 188219392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762387161484453, + "loss": 3.3903, + "theoretical_loss": 4.388490392775209, + "tokens_seen": 188284928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047622868605817455, + "loss": 3.2064, + "theoretical_loss": 4.3882975061910825, + "tokens_seen": 188350464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 114652, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.638829469680786, + "objective/train/theoretical_loss": 4.3881047054941735, + "objective/train/tokens_used": 208876000, + "theoretical_loss": 4.3881047054941735, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762186559679037, + "loss": 3.0819, + "theoretical_loss": 4.3881047054941735, + "tokens_seen": 188416000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004762086258776329, + "loss": 2.914, + "theoretical_loss": 4.387911990616386, + "tokens_seen": 188481536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761985957873621, + "loss": 3.038, + "theoretical_loss": 4.387719361489699, + "tokens_seen": 188547072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761885656970913, + "loss": 2.9676, + "theoretical_loss": 4.3875268180461715, + "tokens_seen": 188612608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047617853560682046, + "loss": 3.2529, + "theoretical_loss": 4.387334360217939, + "tokens_seen": 188678144 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047616850551654964, + "loss": 3.1968, + "theoretical_loss": 4.3871419879372135, + "tokens_seen": 188743680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761584754262788, + "loss": 3.3051, + "theoretical_loss": 4.3869497011362855, + "tokens_seen": 188809216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047614844533600806, + "loss": 3.1021, + "theoretical_loss": 4.386757499747521, + "tokens_seen": 188874752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761384152457372, + "loss": 2.8122, + "theoretical_loss": 4.386565383703366, + "tokens_seen": 188940288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761283851554664, + "loss": 3.308, + "theoretical_loss": 4.38637335293634, + "tokens_seen": 189005824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761183550651956, + "loss": 3.2071, + "theoretical_loss": 4.3861814073790395, + "tokens_seen": 189071360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004761083249749248, + "loss": 3.0571, + "theoretical_loss": 4.38598954696414, + "tokens_seen": 189136896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047609829488465396, + "loss": 3.2339, + "theoretical_loss": 4.385797771624393, + "tokens_seen": 189202432 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047608826479438314, + "loss": 2.9145, + "theoretical_loss": 4.385606081292623, + "tokens_seen": 189267968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760782347041123, + "loss": 2.9952, + "theoretical_loss": 4.385414475901737, + "tokens_seen": 189333504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047606820461384156, + "loss": 3.4306, + "theoretical_loss": 4.385222955384711, + "tokens_seen": 189399040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760581745235707, + "loss": 3.2663, + "theoretical_loss": 4.385031519674601, + "tokens_seen": 189464576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760481444332999, + "loss": 3.267, + "theoretical_loss": 4.38484016870454, + "tokens_seen": 189530112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047603811434302905, + "loss": 3.1402, + "theoretical_loss": 4.384648902407734, + "tokens_seen": 189595648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004760280842527583, + "loss": 3.3339, + "theoretical_loss": 4.384457720717466, + "tokens_seen": 189661184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047601805416248746, + "loss": 3.3079, + "theoretical_loss": 4.384266623567096, + "tokens_seen": 189726720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047600802407221665, + "loss": 3.0988, + "theoretical_loss": 4.384075610890057, + "tokens_seen": 189792256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759979939819458, + "loss": 3.1992, + "theoretical_loss": 4.3838846826198585, + "tokens_seen": 189857792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047598796389167506, + "loss": 3.1131, + "theoretical_loss": 4.383693838690085, + "tokens_seen": 189923328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759779338014042, + "loss": 3.2468, + "theoretical_loss": 4.383503079034397, + "tokens_seen": 189988864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 115651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9877991676330566, + "objective/train/theoretical_loss": 4.383312403586528, + "objective/train/tokens_used": 210514400, + "theoretical_loss": 4.383312403586528, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759679037111334, + "loss": 3.0685, + "theoretical_loss": 4.383312403586528, + "tokens_seen": 190054400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759578736208626, + "loss": 2.9333, + "theoretical_loss": 4.38312181228029, + "tokens_seen": 190119936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759478435305918, + "loss": 3.366, + "theoretical_loss": 4.382931305049566, + "tokens_seen": 190185472 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475937813440321, + "loss": 3.087, + "theoretical_loss": 4.382740881828316, + "tokens_seen": 190251008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047592778335005015, + "loss": 3.0998, + "theoretical_loss": 4.3825505425505735, + "tokens_seen": 190316544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759177532597794, + "loss": 3.172, + "theoretical_loss": 4.382360287150449, + "tokens_seen": 190382080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004759077231695085, + "loss": 3.2168, + "theoretical_loss": 4.382170115562122, + "tokens_seen": 190447616 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047589769307923775, + "loss": 3.2619, + "theoretical_loss": 4.381980027719852, + "tokens_seen": 190513152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047588766298896693, + "loss": 3.1796, + "theoretical_loss": 4.3817900235579685, + "tokens_seen": 190578688 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758776328986961, + "loss": 3.0825, + "theoretical_loss": 4.381600103010879, + "tokens_seen": 190644224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758676028084253, + "loss": 2.9786, + "theoretical_loss": 4.381410266013061, + "tokens_seen": 190709760 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047585757271815447, + "loss": 3.2466, + "theoretical_loss": 4.381220512499068, + "tokens_seen": 190775296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047584754262788365, + "loss": 3.0826, + "theoretical_loss": 4.381030842403527, + "tokens_seen": 190840832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004758375125376129, + "loss": 3.2852, + "theoretical_loss": 4.38084125566114, + "tokens_seen": 190906368 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475827482447342, + "loss": 3.3628, + "theoretical_loss": 4.380651752206679, + "tokens_seen": 190971904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047581745235707125, + "loss": 2.9071, + "theoretical_loss": 4.380462331974991, + "tokens_seen": 191037440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047580742226680043, + "loss": 2.9733, + "theoretical_loss": 4.380272994900998, + "tokens_seen": 191102976 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757973921765296, + "loss": 3.2946, + "theoretical_loss": 4.380083740919693, + "tokens_seen": 191168512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757873620862588, + "loss": 3.0393, + "theoretical_loss": 4.379894569966144, + "tokens_seen": 191234048 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475777331995988, + "loss": 3.2391, + "theoretical_loss": 4.37970548197549, + "tokens_seen": 191299584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047576730190571716, + "loss": 3.0312, + "theoretical_loss": 4.3795164768829435, + "tokens_seen": 191365120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757572718154464, + "loss": 3.0777, + "theoretical_loss": 4.37932755462379, + "tokens_seen": 191430656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757472417251755, + "loss": 3.3053, + "theoretical_loss": 4.379138715133389, + "tokens_seen": 191496192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047573721163490475, + "loss": 3.0129, + "theoretical_loss": 4.378949958347171, + "tokens_seen": 191561728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757271815446339, + "loss": 3.1993, + "theoretical_loss": 4.378761284200638, + "tokens_seen": 191627264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 116251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.686540126800537, + "objective/train/theoretical_loss": 4.3785726926293655, + "objective/train/tokens_used": 212152800, + "theoretical_loss": 4.3785726926293655, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757171514543631, + "loss": 2.8721, + "theoretical_loss": 4.3785726926293655, + "tokens_seen": 191692800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004757071213640923, + "loss": 3.144, + "theoretical_loss": 4.378384183569002, + "tokens_seen": 191758336 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756970912738215, + "loss": 3.2052, + "theoretical_loss": 4.378195756955267, + "tokens_seen": 191823872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047568706118355066, + "loss": 3.192, + "theoretical_loss": 4.378007412723953, + "tokens_seen": 191889408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047567703109327984, + "loss": 2.9825, + "theoretical_loss": 4.377819150810923, + "tokens_seen": 191954944 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475667001003009, + "loss": 2.9849, + "theoretical_loss": 4.377630971152113, + "tokens_seen": 192020480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047565697091273826, + "loss": 3.2592, + "theoretical_loss": 4.377442873683529, + "tokens_seen": 192086016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756469408224674, + "loss": 3.0667, + "theoretical_loss": 4.377254858341252, + "tokens_seen": 192151552 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756369107321966, + "loss": 3.194, + "theoretical_loss": 4.377066925061431, + "tokens_seen": 192217088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004756268806419258, + "loss": 3.2189, + "theoretical_loss": 4.376879073780288, + "tokens_seen": 192282624 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475616850551655, + "loss": 3.0755, + "theoretical_loss": 4.376691304434117, + "tokens_seen": 192348160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047560682046138416, + "loss": 3.1954, + "theoretical_loss": 4.376503616959281, + "tokens_seen": 192413696 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047559679037111334, + "loss": 3.233, + "theoretical_loss": 4.376316011292214, + "tokens_seen": 192479232 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755867602808425, + "loss": 3.0876, + "theoretical_loss": 4.376128487369426, + "tokens_seen": 192544768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047557673019057176, + "loss": 2.9403, + "theoretical_loss": 4.375941045127492, + "tokens_seen": 192610304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755667001003009, + "loss": 3.3601, + "theoretical_loss": 4.375753684503059, + "tokens_seen": 192675840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755566700100301, + "loss": 2.9828, + "theoretical_loss": 4.3755664054328465, + "tokens_seen": 192741376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047554663991975925, + "loss": 2.8375, + "theoretical_loss": 4.375379207853644, + "tokens_seen": 192806912 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004755366098294885, + "loss": 3.323, + "theoretical_loss": 4.375192091702311, + "tokens_seen": 192872448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047552657973921766, + "loss": 3.0783, + "theoretical_loss": 4.375005056915777, + "tokens_seen": 192937984 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047551654964894685, + "loss": 2.974, + "theoretical_loss": 4.374818103431042, + "tokens_seen": 193003520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047550651955867603, + "loss": 3.0758, + "theoretical_loss": 4.374631231185177, + "tokens_seen": 193069056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047549648946840526, + "loss": 3.0464, + "theoretical_loss": 4.374444440115322, + "tokens_seen": 193134592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754864593781344, + "loss": 3.1286, + "theoretical_loss": 4.374257730158687, + "tokens_seen": 193200128 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754764292878636, + "loss": 3.1677, + "theoretical_loss": 4.374071101252554, + "tokens_seen": 193265664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 117431, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7149760723114014, + "objective/train/theoretical_loss": 4.373884553334271, + "objective/train/tokens_used": 213791200, + "theoretical_loss": 4.373884553334271, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047546639919759275, + "loss": 3.1442, + "theoretical_loss": 4.373884553334271, + "tokens_seen": 193331200 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475456369107322, + "loss": 3.1432, + "theoretical_loss": 4.373698086341258, + "tokens_seen": 193396736 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047544633901705117, + "loss": 3.2882, + "theoretical_loss": 4.373511700211004, + "tokens_seen": 193462272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047543630892678035, + "loss": 3.1277, + "theoretical_loss": 4.373325394881069, + "tokens_seen": 193527808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047542627883650953, + "loss": 2.8819, + "theoretical_loss": 4.373139170289079, + "tokens_seen": 193593344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754162487462387, + "loss": 3.1914, + "theoretical_loss": 4.3729530263727305, + "tokens_seen": 193658880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004754062186559679, + "loss": 3.0863, + "theoretical_loss": 4.372766963069793, + "tokens_seen": 193724416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047539618856569713, + "loss": 3.1611, + "theoretical_loss": 4.3725809803181, + "tokens_seen": 193789952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047538615847542625, + "loss": 3.1407, + "theoretical_loss": 4.372395078055554, + "tokens_seen": 193855488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753761283851555, + "loss": 3.2841, + "theoretical_loss": 4.372209256220131, + "tokens_seen": 193921024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753660982948846, + "loss": 3.2507, + "theoretical_loss": 4.3720235147498725, + "tokens_seen": 193986560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047535606820461385, + "loss": 3.2756, + "theoretical_loss": 4.371837853582887, + "tokens_seen": 194052096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047534603811434303, + "loss": 2.9721, + "theoretical_loss": 4.3716522726573555, + "tokens_seen": 194117632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753360080240722, + "loss": 3.1121, + "theoretical_loss": 4.371466771911524, + "tokens_seen": 194183168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004753259779338014, + "loss": 3.07, + "theoretical_loss": 4.371281351283709, + "tokens_seen": 194248704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047531594784353063, + "loss": 3.0433, + "theoretical_loss": 4.371096010712295, + "tokens_seen": 194314240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047530591775325976, + "loss": 3.1014, + "theoretical_loss": 4.370910750135733, + "tokens_seen": 194379776 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475295887662989, + "loss": 3.3776, + "theoretical_loss": 4.370725569492545, + "tokens_seen": 194445312 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752858575727181, + "loss": 3.2206, + "theoretical_loss": 4.370540468721318, + "tokens_seen": 194510848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047527582748244736, + "loss": 3.1334, + "theoretical_loss": 4.370355447760708, + "tokens_seen": 194576384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047526579739217654, + "loss": 3.1063, + "theoretical_loss": 4.370170506549439, + "tokens_seen": 194641920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752557673019057, + "loss": 3.133, + "theoretical_loss": 4.369985645026302, + "tokens_seen": 194707456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752457372116349, + "loss": 3.1316, + "theoretical_loss": 4.369800863130157, + "tokens_seen": 194772992 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752357071213641, + "loss": 3.0326, + "theoretical_loss": 4.36961616079993, + "tokens_seen": 194838528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047522567703109326, + "loss": 3.2395, + "theoretical_loss": 4.3694315379746165, + "tokens_seen": 194904064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 118646, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6308515071868896, + "objective/train/theoretical_loss": 4.369246994593274, + "objective/train/tokens_used": 215429600, + "theoretical_loss": 4.369246994593274, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752156469408225, + "loss": 3.2026, + "theoretical_loss": 4.369246994593274, + "tokens_seen": 194969600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004752056168505517, + "loss": 3.152, + "theoretical_loss": 4.369062530595034, + "tokens_seen": 195035136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047519558676028086, + "loss": 2.9739, + "theoretical_loss": 4.368878145919091, + "tokens_seen": 195100672 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047518555667001004, + "loss": 3.0244, + "theoretical_loss": 4.368693840504706, + "tokens_seen": 195166208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751755265797392, + "loss": 2.816, + "theoretical_loss": 4.368509614291209, + "tokens_seen": 195231744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047516549648946846, + "loss": 3.0789, + "theoretical_loss": 4.368325467217998, + "tokens_seen": 195297280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751554663991976, + "loss": 3.1642, + "theoretical_loss": 4.368141399224534, + "tokens_seen": 195362816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751454363089268, + "loss": 2.9522, + "theoretical_loss": 4.367957410250345, + "tokens_seen": 195428352 + }, + { + "epoch": 0.06, + "learning_rate": 0.000475135406218656, + "loss": 3.0048, + "theoretical_loss": 4.367773500235029, + "tokens_seen": 195493888 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004751253761283852, + "loss": 3.172, + "theoretical_loss": 4.367589669118248, + "tokens_seen": 195559424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047511534603811436, + "loss": 3.02, + "theoretical_loss": 4.367405916839729, + "tokens_seen": 195624960 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047510531594784354, + "loss": 3.0327, + "theoretical_loss": 4.367222243339268, + "tokens_seen": 195690496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750952858575727, + "loss": 3.3324, + "theoretical_loss": 4.367038648556725, + "tokens_seen": 195756032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047508525576730196, + "loss": 3.2043, + "theoretical_loss": 4.366855132432028, + "tokens_seen": 195821568 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750752256770311, + "loss": 3.2153, + "theoretical_loss": 4.366671694905169, + "tokens_seen": 195887104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750651955867603, + "loss": 3.2576, + "theoretical_loss": 4.366488335916207, + "tokens_seen": 195952640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047505516549648945, + "loss": 2.9306, + "theoretical_loss": 4.366305055405267, + "tokens_seen": 196018176 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004750451354062187, + "loss": 3.163, + "theoretical_loss": 4.366121853312537, + "tokens_seen": 196083712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047503510531594787, + "loss": 3.2586, + "theoretical_loss": 4.365938729578274, + "tokens_seen": 196149248 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047502507522567705, + "loss": 3.313, + "theoretical_loss": 4.365755684142799, + "tokens_seen": 196214784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047501504513540623, + "loss": 3.2721, + "theoretical_loss": 4.365572716946499, + "tokens_seen": 196280320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047500501504513546, + "loss": 3.3105, + "theoretical_loss": 4.365389827929825, + "tokens_seen": 196345856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749949849548646, + "loss": 3.2141, + "theoretical_loss": 4.365207017033293, + "tokens_seen": 196411392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749849548645938, + "loss": 3.2766, + "theoretical_loss": 4.365024284197486, + "tokens_seen": 196476928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047497492477432295, + "loss": 2.9995, + "theoretical_loss": 4.364841629363051, + "tokens_seen": 196542464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 119101, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2888681888580322, + "objective/train/theoretical_loss": 4.364659052470699, + "objective/train/tokens_used": 217068000, + "theoretical_loss": 4.364659052470699, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749648946840522, + "loss": 3.1609, + "theoretical_loss": 4.364659052470699, + "tokens_seen": 196608000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047495486459378137, + "loss": 3.1296, + "theoretical_loss": 4.364476553461208, + "tokens_seen": 196673536 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047494483450351055, + "loss": 3.062, + "theoretical_loss": 4.364294132275419, + "tokens_seen": 196739072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047493480441323973, + "loss": 3.1813, + "theoretical_loss": 4.364111788854237, + "tokens_seen": 196804608 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749247743229689, + "loss": 3.0742, + "theoretical_loss": 4.3639295231386335, + "tokens_seen": 196870144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004749147442326981, + "loss": 3.1893, + "theoretical_loss": 4.363747335069642, + "tokens_seen": 196935680 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047490471414242733, + "loss": 3.2753, + "theoretical_loss": 4.363565224588364, + "tokens_seen": 197001216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047489468405215646, + "loss": 3.1104, + "theoretical_loss": 4.363383191635961, + "tokens_seen": 197066752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748846539618857, + "loss": 3.0305, + "theoretical_loss": 4.363201236153663, + "tokens_seen": 197132288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748746238716148, + "loss": 3.0246, + "theoretical_loss": 4.363019358082759, + "tokens_seen": 197197824 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047486459378134405, + "loss": 3.1562, + "theoretical_loss": 4.362837557364608, + "tokens_seen": 197263360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047485456369107323, + "loss": 3.1837, + "theoretical_loss": 4.362655833940627, + "tokens_seen": 197328896 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748445336008024, + "loss": 3.2525, + "theoretical_loss": 4.3624741877523, + "tokens_seen": 197394432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748345035105316, + "loss": 3.1681, + "theoretical_loss": 4.362292618741174, + "tokens_seen": 197459968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047482447342026083, + "loss": 3.4425, + "theoretical_loss": 4.362111126848862, + "tokens_seen": 197525504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047481444332998996, + "loss": 3.3758, + "theoretical_loss": 4.3619297120170355, + "tokens_seen": 197591040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004748044132397192, + "loss": 3.1236, + "theoretical_loss": 4.361748374187433, + "tokens_seen": 197656576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747943831494483, + "loss": 2.9103, + "theoretical_loss": 4.361567113301858, + "tokens_seen": 197722112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047478435305917756, + "loss": 3.3328, + "theoretical_loss": 4.3613859293021715, + "tokens_seen": 197787648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047477432296890674, + "loss": 3.2619, + "theoretical_loss": 4.361204822130304, + "tokens_seen": 197853184 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747642928786359, + "loss": 2.9688, + "theoretical_loss": 4.361023791728243, + "tokens_seen": 197918720 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747542627883651, + "loss": 2.8798, + "theoretical_loss": 4.360842838038045, + "tokens_seen": 197984256 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747442326980943, + "loss": 3.164, + "theoretical_loss": 4.360661961001824, + "tokens_seen": 198049792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047473420260782346, + "loss": 3.2726, + "theoretical_loss": 4.360481160561761, + "tokens_seen": 198115328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747241725175527, + "loss": 3.2962, + "theoretical_loss": 4.360300436660099, + "tokens_seen": 198180864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 119264, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4219841957092285, + "objective/train/theoretical_loss": 4.36011978923914, + "objective/train/tokens_used": 218706400, + "theoretical_loss": 4.36011978923914, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004747141424272818, + "loss": 3.2649, + "theoretical_loss": 4.36011978923914, + "tokens_seen": 198246400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047470411233701106, + "loss": 3.1434, + "theoretical_loss": 4.359939218241253, + "tokens_seen": 198311936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746940822467402, + "loss": 3.3527, + "theoretical_loss": 4.359758723608866, + "tokens_seen": 198377472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746840521564694, + "loss": 3.2825, + "theoretical_loss": 4.359578305284474, + "tokens_seen": 198443008 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746740220661986, + "loss": 3.3552, + "theoretical_loss": 4.359397963210629, + "tokens_seen": 198508544 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746639919759278, + "loss": 3.2728, + "theoretical_loss": 4.3592176973299495, + "tokens_seen": 198574080 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047465396188565696, + "loss": 2.9474, + "theoretical_loss": 4.359037507585111, + "tokens_seen": 198639616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746439317953862, + "loss": 2.9234, + "theoretical_loss": 4.358857393918857, + "tokens_seen": 198705152 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746339017051153, + "loss": 2.9143, + "theoretical_loss": 4.358677356273988, + "tokens_seen": 198770688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047462387161484456, + "loss": 3.1565, + "theoretical_loss": 4.358497394593371, + "tokens_seen": 198836224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746138415245737, + "loss": 3.118, + "theoretical_loss": 4.35831750881993, + "tokens_seen": 198901760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004746038114343029, + "loss": 3.2261, + "theoretical_loss": 4.358137698896653, + "tokens_seen": 198967296 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745937813440321, + "loss": 3.1764, + "theoretical_loss": 4.35795796476659, + "tokens_seen": 199032832 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745837512537613, + "loss": 3.3871, + "theoretical_loss": 4.357778306372852, + "tokens_seen": 199098368 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047457372116349047, + "loss": 3.1078, + "theoretical_loss": 4.357598723658612, + "tokens_seen": 199163904 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047456369107321965, + "loss": 3.0791, + "theoretical_loss": 4.357419216567102, + "tokens_seen": 199229440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047455366098294883, + "loss": 3.2517, + "theoretical_loss": 4.357239785041618, + "tokens_seen": 199294976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047454363089267807, + "loss": 3.176, + "theoretical_loss": 4.357060429025516, + "tokens_seen": 199360512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745336008024072, + "loss": 3.1573, + "theoretical_loss": 4.356881148462213, + "tokens_seen": 199426048 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047452357071213643, + "loss": 2.9391, + "theoretical_loss": 4.356701943295186, + "tokens_seen": 199491584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047451354062186555, + "loss": 3.0178, + "theoretical_loss": 4.356522813467976, + "tokens_seen": 199557120 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004745035105315948, + "loss": 3.008, + "theoretical_loss": 4.356343758924181, + "tokens_seen": 199622656 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047449348044132397, + "loss": 3.088, + "theoretical_loss": 4.356164779607463, + "tokens_seen": 199688192 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047448345035105315, + "loss": 3.0246, + "theoretical_loss": 4.355985875461542, + "tokens_seen": 199753728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744734202607824, + "loss": 3.2933, + "theoretical_loss": 4.3558070464302, + "tokens_seen": 199819264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 120457, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7158637046813965, + "objective/train/theoretical_loss": 4.355628292457278, + "objective/train/tokens_used": 220344800, + "theoretical_loss": 4.355628292457278, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047446339017051157, + "loss": 3.2748, + "theoretical_loss": 4.355628292457278, + "tokens_seen": 199884800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047445336008024075, + "loss": 3.3399, + "theoretical_loss": 4.355449613486681, + "tokens_seen": 199950336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047444332998996993, + "loss": 2.9307, + "theoretical_loss": 4.355271009462371, + "tokens_seen": 200015872 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744332998996991, + "loss": 2.8099, + "theoretical_loss": 4.355092480328369, + "tokens_seen": 200081408 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004744232698094283, + "loss": 3.2346, + "theoretical_loss": 4.35491402602876, + "tokens_seen": 200146944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047441323971915753, + "loss": 3.1045, + "theoretical_loss": 4.354735646507687, + "tokens_seen": 200212480 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047440320962888666, + "loss": 3.1639, + "theoretical_loss": 4.354557341709354, + "tokens_seen": 200278016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743931795386159, + "loss": 3.1702, + "theoretical_loss": 4.354379111578022, + "tokens_seen": 200343552 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474383149448345, + "loss": 3.1362, + "theoretical_loss": 4.3542009560580155, + "tokens_seen": 200409088 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047437311935807425, + "loss": 3.1808, + "theoretical_loss": 4.354022875093716, + "tokens_seen": 200474624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047436308926780343, + "loss": 3.2076, + "theoretical_loss": 4.353844868629566, + "tokens_seen": 200540160 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743530591775326, + "loss": 3.1964, + "theoretical_loss": 4.353666936610068, + "tokens_seen": 200605696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743430290872618, + "loss": 3.0344, + "theoretical_loss": 4.3534890789797815, + "tokens_seen": 200671232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047433299899699103, + "loss": 3.0296, + "theoretical_loss": 4.353311295683328, + "tokens_seen": 200736768 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047432296890672016, + "loss": 3.0211, + "theoretical_loss": 4.353133586665388, + "tokens_seen": 200802304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743129388164494, + "loss": 3.2548, + "theoretical_loss": 4.352955951870699, + "tokens_seen": 200867840 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004743029087261785, + "loss": 3.09, + "theoretical_loss": 4.352778391244061, + "tokens_seen": 200933376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047429287863590776, + "loss": 2.9689, + "theoretical_loss": 4.352600904730329, + "tokens_seen": 200998912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047428284854563694, + "loss": 3.4101, + "theoretical_loss": 4.35242349227442, + "tokens_seen": 201064448 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742728184553661, + "loss": 3.288, + "theoretical_loss": 4.35224615382131, + "tokens_seen": 201129984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742627883650953, + "loss": 3.1146, + "theoretical_loss": 4.352068889316032, + "tokens_seen": 201195520 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742527582748245, + "loss": 3.0871, + "theoretical_loss": 4.351891698703679, + "tokens_seen": 201261056 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047424272818455366, + "loss": 3.2115, + "theoretical_loss": 4.351714581929402, + "tokens_seen": 201326592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742326980942829, + "loss": 2.8483, + "theoretical_loss": 4.351537538938409, + "tokens_seen": 201392128 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474222668004012, + "loss": 3.0209, + "theoretical_loss": 4.351360569675971, + "tokens_seen": 201457664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 121711, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.200042724609375, + "objective/train/theoretical_loss": 4.351183674087412, + "objective/train/tokens_used": 221983200, + "theoretical_loss": 4.351183674087412, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047421263791374126, + "loss": 3.0898, + "theoretical_loss": 4.351183674087412, + "tokens_seen": 201523200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004742026078234704, + "loss": 3.0786, + "theoretical_loss": 4.35100685211812, + "tokens_seen": 201588736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741925777331996, + "loss": 3.155, + "theoretical_loss": 4.350830103713535, + "tokens_seen": 201654272 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741825476429288, + "loss": 3.1987, + "theoretical_loss": 4.350653428819159, + "tokens_seen": 201719808 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474172517552658, + "loss": 3.0476, + "theoretical_loss": 4.350476827380553, + "tokens_seen": 201785344 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047416248746238716, + "loss": 3.157, + "theoretical_loss": 4.350300299343332, + "tokens_seen": 201850880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741524573721164, + "loss": 3.0758, + "theoretical_loss": 4.350123844653173, + "tokens_seen": 201916416 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741424272818455, + "loss": 3.2307, + "theoretical_loss": 4.349947463255808, + "tokens_seen": 201981952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047413239719157476, + "loss": 3.1247, + "theoretical_loss": 4.349771155097029, + "tokens_seen": 202047488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741223671013039, + "loss": 3.1268, + "theoretical_loss": 4.349594920122682, + "tokens_seen": 202113024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741123370110331, + "loss": 3.1097, + "theoretical_loss": 4.349418758278675, + "tokens_seen": 202178560 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004741023069207623, + "loss": 3.139, + "theoretical_loss": 4.349242669510971, + "tokens_seen": 202244096 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740922768304915, + "loss": 3.1241, + "theoretical_loss": 4.3490666537655915, + "tokens_seen": 202309632 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047408224674022067, + "loss": 3.1542, + "theoretical_loss": 4.348890710988615, + "tokens_seen": 202375168 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047407221664994985, + "loss": 3.2895, + "theoretical_loss": 4.348714841126174, + "tokens_seen": 202440704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047406218655967903, + "loss": 3.1676, + "theoretical_loss": 4.348539044124465, + "tokens_seen": 202506240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047405215646940827, + "loss": 2.9643, + "theoretical_loss": 4.348363319929735, + "tokens_seen": 202571776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004740421263791374, + "loss": 3.0266, + "theoretical_loss": 4.348187668488293, + "tokens_seen": 202637312 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047403209628886663, + "loss": 3.0482, + "theoretical_loss": 4.3480120897465016, + "tokens_seen": 202702848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047402206619859575, + "loss": 3.044, + "theoretical_loss": 4.347836583650782, + "tokens_seen": 202768384 + }, + { + "epoch": 0.06, + "learning_rate": 0.000474012036108325, + "loss": 3.1396, + "theoretical_loss": 4.347661150147611, + "tokens_seen": 202833920 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047400200601805417, + "loss": 3.2979, + "theoretical_loss": 4.347485789183523, + "tokens_seen": 202899456 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047399197592778335, + "loss": 3.1927, + "theoretical_loss": 4.34731050070511, + "tokens_seen": 202964992 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047398194583751253, + "loss": 3.0788, + "theoretical_loss": 4.347135284659017, + "tokens_seen": 203030528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047397191574724177, + "loss": 3.0052, + "theoretical_loss": 4.346960140991949, + "tokens_seen": 203096064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 122225, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3157236576080322, + "objective/train/theoretical_loss": 4.346785069650667, + "objective/train/tokens_used": 223621600, + "theoretical_loss": 4.346785069650667, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739618856569709, + "loss": 2.9543, + "theoretical_loss": 4.346785069650667, + "tokens_seen": 203161600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047395185556670013, + "loss": 3.0566, + "theoretical_loss": 4.346610070581987, + "tokens_seen": 203227136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047394182547642926, + "loss": 2.9878, + "theoretical_loss": 4.34643514373278, + "tokens_seen": 203292672 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739317953861585, + "loss": 3.0709, + "theoretical_loss": 4.346260289049978, + "tokens_seen": 203358208 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004739217652958877, + "loss": 3.1681, + "theoretical_loss": 4.346085506480564, + "tokens_seen": 203423744 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047391173520561686, + "loss": 3.2437, + "theoretical_loss": 4.34591079597158, + "tokens_seen": 203489280 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047390170511534604, + "loss": 3.3256, + "theoretical_loss": 4.345736157470122, + "tokens_seen": 203554816 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738916750250752, + "loss": 3.1524, + "theoretical_loss": 4.345561590923343, + "tokens_seen": 203620352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738816449348044, + "loss": 3.3006, + "theoretical_loss": 4.345387096278453, + "tokens_seen": 203685888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047387161484453363, + "loss": 3.088, + "theoretical_loss": 4.345212673482715, + "tokens_seen": 203751424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047386158475426276, + "loss": 3.139, + "theoretical_loss": 4.345038322483449, + "tokens_seen": 203816960 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473851554663992, + "loss": 3.1496, + "theoretical_loss": 4.34486404322803, + "tokens_seen": 203882496 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738415245737211, + "loss": 3.1446, + "theoretical_loss": 4.34468983566389, + "tokens_seen": 203948032 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047383149448345036, + "loss": 3.4082, + "theoretical_loss": 4.3445156997385155, + "tokens_seen": 204013568 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047382146439317954, + "loss": 3.1106, + "theoretical_loss": 4.344341635399447, + "tokens_seen": 204079104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738114343029087, + "loss": 2.9751, + "theoretical_loss": 4.344167642594282, + "tokens_seen": 204144640 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004738014042126379, + "loss": 3.0756, + "theoretical_loss": 4.343993721270673, + "tokens_seen": 204210176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047379137412236714, + "loss": 3.1045, + "theoretical_loss": 4.343819871376326, + "tokens_seen": 204275712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047378134403209626, + "loss": 3.1058, + "theoretical_loss": 4.343646092859004, + "tokens_seen": 204341248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737713139418255, + "loss": 2.8941, + "theoretical_loss": 4.343472385666525, + "tokens_seen": 204406784 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737612838515546, + "loss": 3.2849, + "theoretical_loss": 4.343298749746759, + "tokens_seen": 204472320 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047375125376128386, + "loss": 3.0801, + "theoretical_loss": 4.343125185047634, + "tokens_seen": 204537856 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047374122367101304, + "loss": 3.5135, + "theoretical_loss": 4.342951691517131, + "tokens_seen": 204603392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737311935807422, + "loss": 3.2053, + "theoretical_loss": 4.342778269103286, + "tokens_seen": 204668928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047372116349047146, + "loss": 3.0911, + "theoretical_loss": 4.342604917754191, + "tokens_seen": 204734464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 123173, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.030834436416626, + "objective/train/theoretical_loss": 4.342431637417991, + "objective/train/tokens_used": 225260000, + "theoretical_loss": 4.342431637417991, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737111334002006, + "loss": 3.1185, + "theoretical_loss": 4.342431637417991, + "tokens_seen": 204800000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004737011033099298, + "loss": 3.0803, + "theoretical_loss": 4.342258428042883, + "tokens_seen": 204865536 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473691073219659, + "loss": 3.2108, + "theoretical_loss": 4.342085289577124, + "tokens_seen": 204931072 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736810431293882, + "loss": 3.2812, + "theoretical_loss": 4.341912221969021, + "tokens_seen": 204996608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047367101303911736, + "loss": 3.1871, + "theoretical_loss": 4.341739225166936, + "tokens_seen": 205062144 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736609829488466, + "loss": 3.1366, + "theoretical_loss": 4.341566299119286, + "tokens_seen": 205127680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736509528585757, + "loss": 3.1494, + "theoretical_loss": 4.341393443774542, + "tokens_seen": 205193216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047364092276830496, + "loss": 3.0442, + "theoretical_loss": 4.341220659081228, + "tokens_seen": 205258752 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736308926780341, + "loss": 3.1534, + "theoretical_loss": 4.341047944987921, + "tokens_seen": 205324288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736208625877633, + "loss": 3.1698, + "theoretical_loss": 4.340875301443257, + "tokens_seen": 205389824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736108324974925, + "loss": 3.1333, + "theoretical_loss": 4.340702728395918, + "tokens_seen": 205455360 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004736008024072217, + "loss": 3.1996, + "theoretical_loss": 4.340530225794646, + "tokens_seen": 205520896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047359077231695087, + "loss": 3.3751, + "theoretical_loss": 4.340357793588233, + "tokens_seen": 205586432 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047358074222668005, + "loss": 3.3156, + "theoretical_loss": 4.340185431725527, + "tokens_seen": 205651968 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047357071213640923, + "loss": 3.3006, + "theoretical_loss": 4.3400131401554285, + "tokens_seen": 205717504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047356068204613847, + "loss": 3.1734, + "theoretical_loss": 4.33984091882689, + "tokens_seen": 205783040 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735506519558676, + "loss": 2.9499, + "theoretical_loss": 4.339668767688918, + "tokens_seen": 205848576 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047354062186559683, + "loss": 2.998, + "theoretical_loss": 4.339496686690575, + "tokens_seen": 205914112 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047353059177532595, + "loss": 3.2018, + "theoretical_loss": 4.339324675780972, + "tokens_seen": 205979648 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004735205616850552, + "loss": 3.0006, + "theoretical_loss": 4.339152734909277, + "tokens_seen": 206045184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047351053159478437, + "loss": 3.1032, + "theoretical_loss": 4.338980864024709, + "tokens_seen": 206110720 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047350050150451355, + "loss": 3.1907, + "theoretical_loss": 4.33880906307654, + "tokens_seen": 206176256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047349047141424273, + "loss": 3.1745, + "theoretical_loss": 4.338637332014096, + "tokens_seen": 206241792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047348044132397197, + "loss": 3.1799, + "theoretical_loss": 4.338465670786754, + "tokens_seen": 206307328 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734704112337011, + "loss": 3.1274, + "theoretical_loss": 4.338294079343946, + "tokens_seen": 206372864 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 123915, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9498279094696045, + "objective/train/theoretical_loss": 4.338122557635156, + "objective/train/tokens_used": 226898400, + "theoretical_loss": 4.338122557635156, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047346038114343033, + "loss": 3.1472, + "theoretical_loss": 4.338122557635156, + "tokens_seen": 206438400 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047345035105315946, + "loss": 3.5027, + "theoretical_loss": 4.3379511056099185, + "tokens_seen": 206503936 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734403209628887, + "loss": 3.3358, + "theoretical_loss": 4.337779723217823, + "tokens_seen": 206569472 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734302908726179, + "loss": 3.3308, + "theoretical_loss": 4.33760841040851, + "tokens_seen": 206635008 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047342026078234706, + "loss": 3.2363, + "theoretical_loss": 4.337437167131674, + "tokens_seen": 206700544 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047341023069207624, + "loss": 3.339, + "theoretical_loss": 4.33726599333706, + "tokens_seen": 206766080 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004734002006018054, + "loss": 3.0416, + "theoretical_loss": 4.337094888974467, + "tokens_seen": 206831616 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733901705115346, + "loss": 3.0678, + "theoretical_loss": 4.336923853993743, + "tokens_seen": 206897152 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047338014042126383, + "loss": 3.1236, + "theoretical_loss": 4.336752888344792, + "tokens_seen": 206962688 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047337011033099296, + "loss": 3.1134, + "theoretical_loss": 4.336581991977568, + "tokens_seen": 207028224 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733600802407222, + "loss": 2.9821, + "theoretical_loss": 4.336411164842078, + "tokens_seen": 207093760 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733500501504513, + "loss": 3.0005, + "theoretical_loss": 4.336240406888377, + "tokens_seen": 207159296 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047334002006018056, + "loss": 3.0117, + "theoretical_loss": 4.336069718066579, + "tokens_seen": 207224832 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047332998996990974, + "loss": 2.9441, + "theoretical_loss": 4.335899098326843, + "tokens_seen": 207290368 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733199598796389, + "loss": 3.0821, + "theoretical_loss": 4.335728547619382, + "tokens_seen": 207355904 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004733099297893681, + "loss": 3.1602, + "theoretical_loss": 4.335558065894464, + "tokens_seen": 207421440 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047329989969909734, + "loss": 3.2261, + "theoretical_loss": 4.335387653102402, + "tokens_seen": 207486976 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047328986960882646, + "loss": 3.2569, + "theoretical_loss": 4.335217309193567, + "tokens_seen": 207552512 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732798395185557, + "loss": 3.0878, + "theoretical_loss": 4.335047034118375, + "tokens_seen": 207618048 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732698094282848, + "loss": 3.0245, + "theoretical_loss": 4.334876827827299, + "tokens_seen": 207683584 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047325977933801406, + "loss": 3.0552, + "theoretical_loss": 4.33470669027086, + "tokens_seen": 207749120 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047324974924774324, + "loss": 3.0553, + "theoretical_loss": 4.334536621399632, + "tokens_seen": 207814656 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732397191574724, + "loss": 3.1419, + "theoretical_loss": 4.3343666211642375, + "tokens_seen": 207880192 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732296890672016, + "loss": 2.9697, + "theoretical_loss": 4.334196689515354, + "tokens_seen": 207945728 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004732196589769308, + "loss": 3.4403, + "theoretical_loss": 4.334026826403705, + "tokens_seen": 208011264 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 125279, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2541067600250244, + "objective/train/theoretical_loss": 4.333857031780069, + "objective/train/tokens_used": 228536800, + "theoretical_loss": 4.333857031780069, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047320962888665997, + "loss": 3.1755, + "theoretical_loss": 4.333857031780069, + "tokens_seen": 208076800 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731995987963892, + "loss": 3.3347, + "theoretical_loss": 4.333687305595276, + "tokens_seen": 208142336 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047318956870611833, + "loss": 3.1738, + "theoretical_loss": 4.3335176478002015, + "tokens_seen": 208207872 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047317953861584757, + "loss": 3.0407, + "theoretical_loss": 4.333348058345776, + "tokens_seen": 208273408 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047316950852557675, + "loss": 3.0055, + "theoretical_loss": 4.33317853718298, + "tokens_seen": 208338944 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047315947843530593, + "loss": 3.1805, + "theoretical_loss": 4.333009084262844, + "tokens_seen": 208404480 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731494483450351, + "loss": 2.9499, + "theoretical_loss": 4.33283969953645, + "tokens_seen": 208470016 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731394182547643, + "loss": 3.0383, + "theoretical_loss": 4.332670382954928, + "tokens_seen": 208535552 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047312938816449347, + "loss": 3.1877, + "theoretical_loss": 4.33250113446946, + "tokens_seen": 208601088 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004731193580742227, + "loss": 3.1835, + "theoretical_loss": 4.332331954031279, + "tokens_seen": 208666624 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047310932798395183, + "loss": 3.2001, + "theoretical_loss": 4.332162841591668, + "tokens_seen": 208732160 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047309929789368107, + "loss": 3.2484, + "theoretical_loss": 4.331993797101958, + "tokens_seen": 208797696 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730892678034102, + "loss": 3.37, + "theoretical_loss": 4.331824820513534, + "tokens_seen": 208863232 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047307923771313943, + "loss": 3.1849, + "theoretical_loss": 4.331655911777826, + "tokens_seen": 208928768 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730692076228686, + "loss": 3.219, + "theoretical_loss": 4.331487070846319, + "tokens_seen": 208994304 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730591775325978, + "loss": 3.3387, + "theoretical_loss": 4.331318297670545, + "tokens_seen": 209059840 + }, + { + "epoch": 0.06, + "learning_rate": 0.000473049147442327, + "loss": 3.1956, + "theoretical_loss": 4.3311495922020855, + "tokens_seen": 209125376 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047303911735205615, + "loss": 3.1449, + "theoretical_loss": 4.330980954392573, + "tokens_seen": 209190912 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047302908726178534, + "loss": 2.8264, + "theoretical_loss": 4.33081238419369, + "tokens_seen": 209256448 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047301905717151457, + "loss": 3.0265, + "theoretical_loss": 4.330643881557168, + "tokens_seen": 209321984 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004730090270812437, + "loss": 3.2387, + "theoretical_loss": 4.330475446434788, + "tokens_seen": 209387520 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047299899699097293, + "loss": 3.2265, + "theoretical_loss": 4.330307078778379, + "tokens_seen": 209453056 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729889669007021, + "loss": 3.2092, + "theoretical_loss": 4.330138778539823, + "tokens_seen": 209518592 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729789368104313, + "loss": 3.2283, + "theoretical_loss": 4.329970545671048, + "tokens_seen": 209584128 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047296890672016053, + "loss": 3.2865, + "theoretical_loss": 4.3298023801240335, + "tokens_seen": 209649664 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 125881, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1459879875183105, + "objective/train/theoretical_loss": 4.329634281850807, + "objective/train/tokens_used": 230175200, + "theoretical_loss": 4.329634281850807, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047295887662988966, + "loss": 3.168, + "theoretical_loss": 4.329634281850807, + "tokens_seen": 209715200 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729488465396189, + "loss": 3.1116, + "theoretical_loss": 4.329466250803446, + "tokens_seen": 209780736 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729388164493481, + "loss": 3.1933, + "theoretical_loss": 4.329298286934075, + "tokens_seen": 209846272 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047292878635907726, + "loss": 2.9711, + "theoretical_loss": 4.329130390194871, + "tokens_seen": 209911808 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047291875626880644, + "loss": 3.0108, + "theoretical_loss": 4.3289625605380575, + "tokens_seen": 209977344 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004729087261785356, + "loss": 3.1506, + "theoretical_loss": 4.328794797915908, + "tokens_seen": 210042880 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728986960882648, + "loss": 3.237, + "theoretical_loss": 4.328627102280742, + "tokens_seen": 210108416 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047288866599799403, + "loss": 3.1071, + "theoretical_loss": 4.328459473584933, + "tokens_seen": 210173952 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047287863590772316, + "loss": 3.2433, + "theoretical_loss": 4.328291911780899, + "tokens_seen": 210239488 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728686058174524, + "loss": 3.207, + "theoretical_loss": 4.328124416821107, + "tokens_seen": 210305024 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728585757271815, + "loss": 3.0205, + "theoretical_loss": 4.327956988658076, + "tokens_seen": 210370560 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047284854563691076, + "loss": 3.2139, + "theoretical_loss": 4.3277896272443686, + "tokens_seen": 210436096 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047283851554663994, + "loss": 3.0787, + "theoretical_loss": 4.3276223325325995, + "tokens_seen": 210501632 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728284854563691, + "loss": 3.1398, + "theoretical_loss": 4.327455104475431, + "tokens_seen": 210567168 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004728184553660983, + "loss": 3.3631, + "theoretical_loss": 4.327287943025572, + "tokens_seen": 210632704 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047280842527582754, + "loss": 2.9762, + "theoretical_loss": 4.327120848135781, + "tokens_seen": 210698240 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047279839518555666, + "loss": 3.067, + "theoretical_loss": 4.326953819758867, + "tokens_seen": 210763776 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727883650952859, + "loss": 3.2263, + "theoretical_loss": 4.326786857847681, + "tokens_seen": 210829312 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472778335005015, + "loss": 3.3092, + "theoretical_loss": 4.326619962355128, + "tokens_seen": 210894848 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047276830491474426, + "loss": 3.4328, + "theoretical_loss": 4.326453133234159, + "tokens_seen": 210960384 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047275827482447344, + "loss": 3.1644, + "theoretical_loss": 4.326286370437772, + "tokens_seen": 211025920 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727482447342026, + "loss": 3.1984, + "theoretical_loss": 4.326119673919013, + "tokens_seen": 211091456 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727382146439318, + "loss": 3.0863, + "theoretical_loss": 4.325953043630978, + "tokens_seen": 211156992 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472728184553661, + "loss": 3.0964, + "theoretical_loss": 4.3257864795268075, + "tokens_seen": 211222528 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047271815446339017, + "loss": 3.0099, + "theoretical_loss": 4.325619981559692, + "tokens_seen": 211288064 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 127261, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0648367404937744, + "objective/train/theoretical_loss": 4.325453549682869, + "objective/train/tokens_used": 231813600, + "theoretical_loss": 4.325453549682869, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004727081243731194, + "loss": 3.116, + "theoretical_loss": 4.325453549682869, + "tokens_seen": 211353600 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047269809428284853, + "loss": 3.0499, + "theoretical_loss": 4.325287183849625, + "tokens_seen": 211419136 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047268806419257777, + "loss": 3.0799, + "theoretical_loss": 4.325120884013289, + "tokens_seen": 211484672 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047267803410230695, + "loss": 3.1633, + "theoretical_loss": 4.324954650127244, + "tokens_seen": 211550208 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047266800401203613, + "loss": 2.8267, + "theoretical_loss": 4.324788482144916, + "tokens_seen": 211615744 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726579739217653, + "loss": 2.9392, + "theoretical_loss": 4.324622380019778, + "tokens_seen": 211681280 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726479438314945, + "loss": 3.1858, + "theoretical_loss": 4.3244563437053545, + "tokens_seen": 211746816 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047263791374122367, + "loss": 3.1446, + "theoretical_loss": 4.324290373155213, + "tokens_seen": 211812352 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004726278836509529, + "loss": 3.259, + "theoretical_loss": 4.324124468322971, + "tokens_seen": 211877888 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047261785356068203, + "loss": 3.195, + "theoretical_loss": 4.32395862916229, + "tokens_seen": 211943424 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047260782347041127, + "loss": 3.0971, + "theoretical_loss": 4.32379285562688, + "tokens_seen": 212008960 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725977933801404, + "loss": 3.1254, + "theoretical_loss": 4.323627147670498, + "tokens_seen": 212074496 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047258776328986963, + "loss": 3.0839, + "theoretical_loss": 4.323461505246949, + "tokens_seen": 212140032 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725777331995988, + "loss": 2.8079, + "theoretical_loss": 4.323295928310084, + "tokens_seen": 212205568 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472567703109328, + "loss": 3.1229, + "theoretical_loss": 4.3231304168137985, + "tokens_seen": 212271104 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725576730190572, + "loss": 3.1019, + "theoretical_loss": 4.322964970712039, + "tokens_seen": 212336640 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047254764292878636, + "loss": 3.272, + "theoretical_loss": 4.322799589958794, + "tokens_seen": 212402176 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047253761283851554, + "loss": 2.7217, + "theoretical_loss": 4.3226342745081014, + "tokens_seen": 212467712 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047252758274824477, + "loss": 3.0247, + "theoretical_loss": 4.322469024314046, + "tokens_seen": 212533248 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004725175526579739, + "loss": 3.2237, + "theoretical_loss": 4.322303839330757, + "tokens_seen": 212598784 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047250752256770313, + "loss": 3.3425, + "theoretical_loss": 4.322138719512411, + "tokens_seen": 212664320 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724974924774323, + "loss": 3.2206, + "theoretical_loss": 4.321973664813231, + "tokens_seen": 212729856 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724874623871615, + "loss": 3.2825, + "theoretical_loss": 4.321808675187486, + "tokens_seen": 212795392 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724774322968907, + "loss": 2.8949, + "theoretical_loss": 4.321643750589493, + "tokens_seen": 212860928 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047246740220661986, + "loss": 3.0597, + "theoretical_loss": 4.32147889097361, + "tokens_seen": 212926464 + }, + { + "epoch": 0.06, + "objective/train/docs_used": 127839, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5590968132019043, + "objective/train/theoretical_loss": 4.321314096294248, + "objective/train/tokens_used": 233452000, + "theoretical_loss": 4.321314096294248, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047245737211634904, + "loss": 2.9514, + "theoretical_loss": 4.321314096294248, + "tokens_seen": 212992000 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724473420260783, + "loss": 3.0759, + "theoretical_loss": 4.321149366505858, + "tokens_seen": 213057536 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004724373119358074, + "loss": 2.9617, + "theoretical_loss": 4.32098470156294, + "tokens_seen": 213123072 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047242728184553664, + "loss": 2.9706, + "theoretical_loss": 4.320820101420041, + "tokens_seen": 213188608 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047241725175526576, + "loss": 3.3507, + "theoretical_loss": 4.320655566031751, + "tokens_seen": 213254144 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472407221664995, + "loss": 3.0787, + "theoretical_loss": 4.320491095352706, + "tokens_seen": 213319680 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723971915747242, + "loss": 3.2185, + "theoretical_loss": 4.320326689337589, + "tokens_seen": 213385216 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047238716148445336, + "loss": 3.2852, + "theoretical_loss": 4.320162347941129, + "tokens_seen": 213450752 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047237713139418254, + "loss": 3.0982, + "theoretical_loss": 4.3199980711181, + "tokens_seen": 213516288 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723671013039117, + "loss": 3.1012, + "theoretical_loss": 4.319833858823321, + "tokens_seen": 213581824 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723570712136409, + "loss": 2.945, + "theoretical_loss": 4.319669711011656, + "tokens_seen": 213647360 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047234704112337014, + "loss": 3.1732, + "theoretical_loss": 4.319505627638016, + "tokens_seen": 213712896 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047233701103309927, + "loss": 3.1215, + "theoretical_loss": 4.319341608657357, + "tokens_seen": 213778432 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723269809428285, + "loss": 3.3584, + "theoretical_loss": 4.319177654024679, + "tokens_seen": 213843968 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004723169508525577, + "loss": 3.0733, + "theoretical_loss": 4.319013763695029, + "tokens_seen": 213909504 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047230692076228686, + "loss": 3.2797, + "theoretical_loss": 4.318849937623497, + "tokens_seen": 213975040 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047229689067201605, + "loss": 2.9947, + "theoretical_loss": 4.318686175765221, + "tokens_seen": 214040576 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722868605817452, + "loss": 3.0984, + "theoretical_loss": 4.31852247807538, + "tokens_seen": 214106112 + }, + { + "epoch": 0.06, + "learning_rate": 0.0004722768304914744, + "loss": 2.8851, + "theoretical_loss": 4.318358844509202, + "tokens_seen": 214171648 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047226680040120364, + "loss": 3.0447, + "theoretical_loss": 4.318195275021959, + "tokens_seen": 214237184 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047225677031093277, + "loss": 3.2885, + "theoretical_loss": 4.318031769568966, + "tokens_seen": 214302720 + }, + { + "epoch": 0.06, + "learning_rate": 0.000472246740220662, + "loss": 3.0849, + "theoretical_loss": 4.317868328105585, + "tokens_seen": 214368256 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047223671013039113, + "loss": 3.052, + "theoretical_loss": 4.3177049505872205, + "tokens_seen": 214433792 + }, + { + "epoch": 0.06, + "learning_rate": 0.00047222668004012037, + "loss": 3.0657, + "theoretical_loss": 4.317541636969324, + "tokens_seen": 214499328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004722166499498496, + "loss": 2.9079, + "theoretical_loss": 4.31737838720739, + "tokens_seen": 214564864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 128852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.337009906768799, + "objective/train/theoretical_loss": 4.317215201256958, + "objective/train/tokens_used": 235090400, + "theoretical_loss": 4.317215201256958, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047220661985957873, + "loss": 3.1137, + "theoretical_loss": 4.317215201256958, + "tokens_seen": 214630400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047219658976930797, + "loss": 3.0925, + "theoretical_loss": 4.317052079073613, + "tokens_seen": 214695936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047218655967903715, + "loss": 3.1037, + "theoretical_loss": 4.316889020612983, + "tokens_seen": 214761472 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047217652958876633, + "loss": 2.9058, + "theoretical_loss": 4.316726025830741, + "tokens_seen": 214827008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721664994984955, + "loss": 3.1171, + "theoretical_loss": 4.316563094682604, + "tokens_seen": 214892544 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721564694082247, + "loss": 3.2248, + "theoretical_loss": 4.316400227124335, + "tokens_seen": 214958080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047214643931795387, + "loss": 3.1588, + "theoretical_loss": 4.3162374231117395, + "tokens_seen": 215023616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721364092276831, + "loss": 3.2238, + "theoretical_loss": 4.316074682600666, + "tokens_seen": 215089152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047212637913741223, + "loss": 3.3115, + "theoretical_loss": 4.315912005547011, + "tokens_seen": 215154688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047211634904714147, + "loss": 2.9668, + "theoretical_loss": 4.315749391906712, + "tokens_seen": 215220224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004721063189568706, + "loss": 3.0336, + "theoretical_loss": 4.31558684163575, + "tokens_seen": 215285760 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047209628886659983, + "loss": 3.1727, + "theoretical_loss": 4.315424354690153, + "tokens_seen": 215351296 + }, + { + "epoch": 0.07, + "learning_rate": 0.000472086258776329, + "loss": 3.0615, + "theoretical_loss": 4.31526193102599, + "tokens_seen": 215416832 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720762286860582, + "loss": 2.9988, + "theoretical_loss": 4.315099570599374, + "tokens_seen": 215482368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720661985957874, + "loss": 3.0711, + "theoretical_loss": 4.314937273366466, + "tokens_seen": 215547904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047205616850551656, + "loss": 3.2474, + "theoretical_loss": 4.314775039283463, + "tokens_seen": 215613440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047204613841524574, + "loss": 3.2283, + "theoretical_loss": 4.314612868306614, + "tokens_seen": 215678976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047203610832497497, + "loss": 3.1768, + "theoretical_loss": 4.314450760392204, + "tokens_seen": 215744512 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720260782347041, + "loss": 3.23, + "theoretical_loss": 4.314288715496568, + "tokens_seen": 215810048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047201604814443333, + "loss": 3.443, + "theoretical_loss": 4.314126733576082, + "tokens_seen": 215875584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004720060180541625, + "loss": 2.9834, + "theoretical_loss": 4.3139648145871625, + "tokens_seen": 215941120 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719959879638917, + "loss": 3.3007, + "theoretical_loss": 4.313802958486274, + "tokens_seen": 216006656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719859578736209, + "loss": 3.2257, + "theoretical_loss": 4.313641165229922, + "tokens_seen": 216072192 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047197592778335006, + "loss": 2.9371, + "theoretical_loss": 4.313479434774655, + "tokens_seen": 216137728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047196589769307924, + "loss": 3.1597, + "theoretical_loss": 4.313317767077065, + "tokens_seen": 216203264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 129467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.151235342025757, + "objective/train/theoretical_loss": 4.3131561620937875, + "objective/train/tokens_used": 236728800, + "theoretical_loss": 4.3131561620937875, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719558676028085, + "loss": 3.1703, + "theoretical_loss": 4.3131561620937875, + "tokens_seen": 216268800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719458375125376, + "loss": 3.1118, + "theoretical_loss": 4.312994619781502, + "tokens_seen": 216334336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047193580742226684, + "loss": 3.2158, + "theoretical_loss": 4.312833140096929, + "tokens_seen": 216399872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047192577733199596, + "loss": 3.1632, + "theoretical_loss": 4.312671722996835, + "tokens_seen": 216465408 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719157472417252, + "loss": 3.0144, + "theoretical_loss": 4.312510368438025, + "tokens_seen": 216530944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004719057171514544, + "loss": 2.9652, + "theoretical_loss": 4.31234907637735, + "tokens_seen": 216596480 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047189568706118356, + "loss": 3.0804, + "theoretical_loss": 4.312187846771704, + "tokens_seen": 216662016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047188565697091274, + "loss": 3.221, + "theoretical_loss": 4.312026679578022, + "tokens_seen": 216727552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718756268806419, + "loss": 3.2019, + "theoretical_loss": 4.311865574753283, + "tokens_seen": 216793088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718655967903711, + "loss": 2.9076, + "theoretical_loss": 4.311704532254509, + "tokens_seen": 216858624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047185556670010034, + "loss": 3.1173, + "theoretical_loss": 4.311543552038762, + "tokens_seen": 216924160 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047184553660982947, + "loss": 2.9789, + "theoretical_loss": 4.31138263406315, + "tokens_seen": 216989696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718355065195587, + "loss": 3.0793, + "theoretical_loss": 4.311221778284821, + "tokens_seen": 217055232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004718254764292879, + "loss": 3.1944, + "theoretical_loss": 4.311060984660967, + "tokens_seen": 217120768 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047181544633901706, + "loss": 2.9635, + "theoretical_loss": 4.310900253148821, + "tokens_seen": 217186304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047180541624874625, + "loss": 3.0845, + "theoretical_loss": 4.31073958370566, + "tokens_seen": 217251840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717953861584754, + "loss": 3.3273, + "theoretical_loss": 4.3105789762888005, + "tokens_seen": 217317376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717853560682046, + "loss": 3.1426, + "theoretical_loss": 4.310418430855604, + "tokens_seen": 217382912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047177532597793384, + "loss": 3.1465, + "theoretical_loss": 4.310257947363474, + "tokens_seen": 217448448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047176529588766297, + "loss": 3.1651, + "theoretical_loss": 4.310097525769854, + "tokens_seen": 217513984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717552657973922, + "loss": 3.406, + "theoretical_loss": 4.309937166032231, + "tokens_seen": 217579520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047174523570712133, + "loss": 3.3688, + "theoretical_loss": 4.309776868108134, + "tokens_seen": 217645056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047173520561685057, + "loss": 3.2583, + "theoretical_loss": 4.309616631955134, + "tokens_seen": 217710592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047172517552657975, + "loss": 3.2181, + "theoretical_loss": 4.309456457530843, + "tokens_seen": 217776128 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047171514543630893, + "loss": 3.0626, + "theoretical_loss": 4.309296344792916, + "tokens_seen": 217841664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 130589, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.174252510070801, + "objective/train/theoretical_loss": 4.309136293699048, + "objective/train/tokens_used": 238367200, + "theoretical_loss": 4.309136293699048, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004717051153460381, + "loss": 3.0759, + "theoretical_loss": 4.309136293699048, + "tokens_seen": 217907200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047169508525576735, + "loss": 3.0866, + "theoretical_loss": 4.308976304206978, + "tokens_seen": 217972736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716850551654965, + "loss": 3.0519, + "theoretical_loss": 4.3088163762744855, + "tokens_seen": 218038272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716750250752257, + "loss": 2.938, + "theoretical_loss": 4.308656509859392, + "tokens_seen": 218103808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047166499498495484, + "loss": 2.9371, + "theoretical_loss": 4.308496704919559, + "tokens_seen": 218169344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047165496489468407, + "loss": 2.8471, + "theoretical_loss": 4.308336961412892, + "tokens_seen": 218234880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047164493480441325, + "loss": 3.1918, + "theoretical_loss": 4.308177279297336, + "tokens_seen": 218300416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047163490471414243, + "loss": 3.0599, + "theoretical_loss": 4.308017658530879, + "tokens_seen": 218365952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716248746238716, + "loss": 3.1856, + "theoretical_loss": 4.307858099071549, + "tokens_seen": 218431488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004716148445336008, + "loss": 2.7223, + "theoretical_loss": 4.307698600877416, + "tokens_seen": 218497024 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047160481444333, + "loss": 3.1367, + "theoretical_loss": 4.30753916390659, + "tokens_seen": 218562560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715947843530592, + "loss": 3.048, + "theoretical_loss": 4.307379788117224, + "tokens_seen": 218628096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047158475426278834, + "loss": 3.1323, + "theoretical_loss": 4.3072204734675115, + "tokens_seen": 218693632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715747241725176, + "loss": 2.804, + "theoretical_loss": 4.307061219915687, + "tokens_seen": 218759168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715646940822467, + "loss": 3.1227, + "theoretical_loss": 4.306902027420025, + "tokens_seen": 218824704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047155466399197594, + "loss": 3.3008, + "theoretical_loss": 4.306742895938844, + "tokens_seen": 218890240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715446339017051, + "loss": 3.1554, + "theoretical_loss": 4.306583825430499, + "tokens_seen": 218955776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715346038114343, + "loss": 2.8274, + "theoretical_loss": 4.30642481585339, + "tokens_seen": 219021312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715245737211635, + "loss": 3.1366, + "theoretical_loss": 4.306265867165955, + "tokens_seen": 219086848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004715145436308927, + "loss": 2.9215, + "theoretical_loss": 4.306106979326674, + "tokens_seen": 219152384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047150451354062184, + "loss": 3.2813, + "theoretical_loss": 4.305948152294068, + "tokens_seen": 219217920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714944834503511, + "loss": 2.9276, + "theoretical_loss": 4.305789386026698, + "tokens_seen": 219283456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714844533600802, + "loss": 3.0313, + "theoretical_loss": 4.305630680483166, + "tokens_seen": 219348992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047147442326980944, + "loss": 3.2763, + "theoretical_loss": 4.305472035622114, + "tokens_seen": 219414528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714643931795387, + "loss": 3.1934, + "theoretical_loss": 4.305313451402226, + "tokens_seen": 219480064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 131942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.265974283218384, + "objective/train/theoretical_loss": 4.305154927782223, + "objective/train/tokens_used": 240005600, + "theoretical_loss": 4.305154927782223, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714543630892678, + "loss": 3.0968, + "theoretical_loss": 4.305154927782223, + "tokens_seen": 219545600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047144433299899704, + "loss": 3.047, + "theoretical_loss": 4.304996464720872, + "tokens_seen": 219611136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047143430290872616, + "loss": 3.1081, + "theoretical_loss": 4.304838062176974, + "tokens_seen": 219676672 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714242728184554, + "loss": 3.1186, + "theoretical_loss": 4.304679720109376, + "tokens_seen": 219742208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004714142427281846, + "loss": 3.161, + "theoretical_loss": 4.304521438476961, + "tokens_seen": 219807744 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047140421263791376, + "loss": 3.122, + "theoretical_loss": 4.304363217238654, + "tokens_seen": 219873280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047139418254764294, + "loss": 3.1317, + "theoretical_loss": 4.304205056353421, + "tokens_seen": 219938816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713841524573721, + "loss": 3.0803, + "theoretical_loss": 4.304046955780267, + "tokens_seen": 220004352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713741223671013, + "loss": 3.1409, + "theoretical_loss": 4.303888915478236, + "tokens_seen": 220069888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047136409227683054, + "loss": 3.1707, + "theoretical_loss": 4.303730935406414, + "tokens_seen": 220135424 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047135406218655967, + "loss": 2.9751, + "theoretical_loss": 4.303573015523927, + "tokens_seen": 220200960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713440320962889, + "loss": 3.177, + "theoretical_loss": 4.3034151557899385, + "tokens_seen": 220266496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004713340020060181, + "loss": 3.054, + "theoretical_loss": 4.303257356163654, + "tokens_seen": 220332032 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047132397191574726, + "loss": 3.4153, + "theoretical_loss": 4.303099616604318, + "tokens_seen": 220397568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047131394182547645, + "loss": 3.0444, + "theoretical_loss": 4.302941937071216, + "tokens_seen": 220463104 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047130391173520563, + "loss": 2.9421, + "theoretical_loss": 4.3027843175236695, + "tokens_seen": 220528640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712938816449348, + "loss": 3.1708, + "theoretical_loss": 4.302626757921045, + "tokens_seen": 220594176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047128385155466404, + "loss": 3.3514, + "theoretical_loss": 4.302469258222744, + "tokens_seen": 220659712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047127382146439317, + "loss": 3.2391, + "theoretical_loss": 4.30231181838821, + "tokens_seen": 220725248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712637913741224, + "loss": 3.122, + "theoretical_loss": 4.302154438376926, + "tokens_seen": 220790784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047125376128385153, + "loss": 3.0711, + "theoretical_loss": 4.301997118148412, + "tokens_seen": 220856320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047124373119358077, + "loss": 3.2042, + "theoretical_loss": 4.301839857662231, + "tokens_seen": 220921856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047123370110330995, + "loss": 2.8998, + "theoretical_loss": 4.301682656877983, + "tokens_seen": 220987392 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047122367101303913, + "loss": 3.3909, + "theoretical_loss": 4.301525515755307, + "tokens_seen": 221052928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004712136409227683, + "loss": 3.0117, + "theoretical_loss": 4.301368434253883, + "tokens_seen": 221118464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 132442, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3066797256469727, + "objective/train/theoretical_loss": 4.301211412333429, + "objective/train/tokens_used": 241644000, + "theoretical_loss": 4.301211412333429, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047120361083249755, + "loss": 3.1236, + "theoretical_loss": 4.301211412333429, + "tokens_seen": 221184000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711935807422267, + "loss": 3.0539, + "theoretical_loss": 4.301054449953702, + "tokens_seen": 221249536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711835506519559, + "loss": 3.2493, + "theoretical_loss": 4.300897547074499, + "tokens_seen": 221315072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047117352056168504, + "loss": 3.494, + "theoretical_loss": 4.300740703655655, + "tokens_seen": 221380608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047116349047141427, + "loss": 2.8181, + "theoretical_loss": 4.300583919657045, + "tokens_seen": 221446144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047115346038114345, + "loss": 3.1062, + "theoretical_loss": 4.300427195038582, + "tokens_seen": 221511680 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047114343029087263, + "loss": 3.2018, + "theoretical_loss": 4.300270529760219, + "tokens_seen": 221577216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711334002006018, + "loss": 2.9325, + "theoretical_loss": 4.300113923781946, + "tokens_seen": 221642752 + }, + { + "epoch": 0.07, + "learning_rate": 0.000471123370110331, + "loss": 3.0023, + "theoretical_loss": 4.2999573770637936, + "tokens_seen": 221708288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711133400200602, + "loss": 3.1196, + "theoretical_loss": 4.299800889565831, + "tokens_seen": 221773824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004711033099297894, + "loss": 3.0607, + "theoretical_loss": 4.299644461248165, + "tokens_seen": 221839360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047109327983951854, + "loss": 3.2191, + "theoretical_loss": 4.299488092070942, + "tokens_seen": 221904896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710832497492478, + "loss": 3.0538, + "theoretical_loss": 4.299331781994347, + "tokens_seen": 221970432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710732196589769, + "loss": 3.073, + "theoretical_loss": 4.2991755309786015, + "tokens_seen": 222035968 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047106318956870614, + "loss": 2.9409, + "theoretical_loss": 4.299019338983969, + "tokens_seen": 222101504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710531594784353, + "loss": 3.0287, + "theoretical_loss": 4.298863205970751, + "tokens_seen": 222167040 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710431293881645, + "loss": 3.1599, + "theoretical_loss": 4.298707131899281, + "tokens_seen": 222232576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710330992978937, + "loss": 3.146, + "theoretical_loss": 4.298551116729941, + "tokens_seen": 222298112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710230692076229, + "loss": 3.085, + "theoretical_loss": 4.298395160423144, + "tokens_seen": 222363648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047101303911735204, + "loss": 3.09, + "theoretical_loss": 4.298239262939344, + "tokens_seen": 222429184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004710030090270813, + "loss": 3.0419, + "theoretical_loss": 4.2980834242390324, + "tokens_seen": 222494720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709929789368104, + "loss": 3.2707, + "theoretical_loss": 4.29792764428274, + "tokens_seen": 222560256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047098294884653964, + "loss": 3.2489, + "theoretical_loss": 4.297771923031033, + "tokens_seen": 222625792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709729187562688, + "loss": 2.9268, + "theoretical_loss": 4.297616260444519, + "tokens_seen": 222691328 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470962888665998, + "loss": 3.1398, + "theoretical_loss": 4.297460656483842, + "tokens_seen": 222756864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 133596, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.741494655609131, + "objective/train/theoretical_loss": 4.297305111109683, + "objective/train/tokens_used": 243282400, + "theoretical_loss": 4.297305111109683, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709528585757272, + "loss": 2.7095, + "theoretical_loss": 4.297305111109683, + "tokens_seen": 222822400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047094282848545636, + "loss": 2.9503, + "theoretical_loss": 4.297149624282762, + "tokens_seen": 222887936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047093279839518555, + "loss": 3.1134, + "theoretical_loss": 4.296994195963838, + "tokens_seen": 222953472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709227683049148, + "loss": 2.8174, + "theoretical_loss": 4.296838826113706, + "tokens_seen": 223019008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004709127382146439, + "loss": 2.7694, + "theoretical_loss": 4.296683514693199, + "tokens_seen": 223084544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047090270812437314, + "loss": 3.1462, + "theoretical_loss": 4.296528261663189, + "tokens_seen": 223150080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047089267803410227, + "loss": 3.1611, + "theoretical_loss": 4.296373066984585, + "tokens_seen": 223215616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708826479438315, + "loss": 3.0784, + "theoretical_loss": 4.296217930618332, + "tokens_seen": 223281152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708726178535607, + "loss": 3.2461, + "theoretical_loss": 4.2960628525254165, + "tokens_seen": 223346688 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047086258776328987, + "loss": 3.3592, + "theoretical_loss": 4.295907832666858, + "tokens_seen": 223412224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047085255767301905, + "loss": 3.134, + "theoretical_loss": 4.2957528710037165, + "tokens_seen": 223477760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708425275827483, + "loss": 2.793, + "theoretical_loss": 4.2955979674970886, + "tokens_seen": 223543296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708324974924774, + "loss": 3.1087, + "theoretical_loss": 4.295443122108108, + "tokens_seen": 223608832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047082246740220665, + "loss": 3.2745, + "theoretical_loss": 4.295288334797946, + "tokens_seen": 223674368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004708124373119358, + "loss": 3.0884, + "theoretical_loss": 4.295133605527812, + "tokens_seen": 223739904 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470802407221665, + "loss": 3.0021, + "theoretical_loss": 4.294978934258951, + "tokens_seen": 223805440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707923771313942, + "loss": 2.9157, + "theoretical_loss": 4.294824320952646, + "tokens_seen": 223870976 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047078234704112337, + "loss": 3.3173, + "theoretical_loss": 4.294669765570219, + "tokens_seen": 223936512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047077231695085255, + "loss": 3.1127, + "theoretical_loss": 4.294515268073026, + "tokens_seen": 224002048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047076228686058173, + "loss": 2.8344, + "theoretical_loss": 4.294360828422461, + "tokens_seen": 224067584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707522567703109, + "loss": 3.0905, + "theoretical_loss": 4.294206446579956, + "tokens_seen": 224133120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047074222668004015, + "loss": 3.1892, + "theoretical_loss": 4.29405212250698, + "tokens_seen": 224198656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707321965897693, + "loss": 2.8706, + "theoretical_loss": 4.293897856165039, + "tokens_seen": 224264192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707221664994985, + "loss": 3.2766, + "theoretical_loss": 4.293743647515674, + "tokens_seen": 224329728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047071213640922775, + "loss": 3.1984, + "theoretical_loss": 4.293589496520464, + "tokens_seen": 224395264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 134273, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.291004180908203, + "objective/train/theoretical_loss": 4.293435403141026, + "objective/train/tokens_used": 244920800, + "theoretical_loss": 4.293435403141026, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004707021063189569, + "loss": 3.2134, + "theoretical_loss": 4.293435403141026, + "tokens_seen": 224460800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706920762286861, + "loss": 3.107, + "theoretical_loss": 4.293281367339013, + "tokens_seen": 224526336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047068204613841524, + "loss": 3.2989, + "theoretical_loss": 4.293127389076113, + "tokens_seen": 224591872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047067201604814447, + "loss": 3.4678, + "theoretical_loss": 4.292973468314053, + "tokens_seen": 224657408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047066198595787365, + "loss": 3.0958, + "theoretical_loss": 4.292819605014596, + "tokens_seen": 224722944 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047065195586760283, + "loss": 2.9754, + "theoretical_loss": 4.292665799139541, + "tokens_seen": 224788480 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470641925777332, + "loss": 3.2233, + "theoretical_loss": 4.292512050650723, + "tokens_seen": 224854016 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706318956870612, + "loss": 2.9816, + "theoretical_loss": 4.292358359510015, + "tokens_seen": 224919552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706218655967904, + "loss": 3.2302, + "theoretical_loss": 4.292204725679326, + "tokens_seen": 224985088 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004706118355065196, + "loss": 3.1335, + "theoretical_loss": 4.2920511491206, + "tokens_seen": 225050624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047060180541624874, + "loss": 3.1664, + "theoretical_loss": 4.2918976297958205, + "tokens_seen": 225116160 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470591775325978, + "loss": 3.0454, + "theoretical_loss": 4.2917441676670025, + "tokens_seen": 225181696 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705817452357071, + "loss": 3.0739, + "theoretical_loss": 4.291590762696202, + "tokens_seen": 225247232 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047057171514543634, + "loss": 3.1721, + "theoretical_loss": 4.291437414845509, + "tokens_seen": 225312768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705616850551655, + "loss": 3.1347, + "theoretical_loss": 4.291284124077049, + "tokens_seen": 225378304 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705516549648947, + "loss": 2.9492, + "theoretical_loss": 4.291130890352985, + "tokens_seen": 225443840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705416248746239, + "loss": 3.1897, + "theoretical_loss": 4.290977713635515, + "tokens_seen": 225509376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705315947843531, + "loss": 3.2739, + "theoretical_loss": 4.2908245938868745, + "tokens_seen": 225574912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047052156469408224, + "loss": 3.3183, + "theoretical_loss": 4.290671531069334, + "tokens_seen": 225640448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705115346038115, + "loss": 3.1922, + "theoretical_loss": 4.290518525145199, + "tokens_seen": 225705984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004705015045135406, + "loss": 3.282, + "theoretical_loss": 4.290365576076812, + "tokens_seen": 225771520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047049147442326984, + "loss": 3.2065, + "theoretical_loss": 4.290212683826551, + "tokens_seen": 225837056 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470481444332999, + "loss": 3.251, + "theoretical_loss": 4.290059848356833, + "tokens_seen": 225902592 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704714142427282, + "loss": 2.9908, + "theoretical_loss": 4.289907069630104, + "tokens_seen": 225968128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704613841524574, + "loss": 3.2375, + "theoretical_loss": 4.28975434760885, + "tokens_seen": 226033664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 135613, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.803276538848877, + "objective/train/theoretical_loss": 4.2896016822555945, + "objective/train/tokens_used": 246559200, + "theoretical_loss": 4.2896016822555945, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047045135406218656, + "loss": 3.0062, + "theoretical_loss": 4.2896016822555945, + "tokens_seen": 226099200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047044132397191575, + "loss": 3.1042, + "theoretical_loss": 4.289449073532892, + "tokens_seen": 226164736 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470431293881645, + "loss": 3.3344, + "theoretical_loss": 4.289296521403337, + "tokens_seen": 226230272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004704212637913741, + "loss": 3.1889, + "theoretical_loss": 4.289144025829555, + "tokens_seen": 226295808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047041123370110334, + "loss": 2.9189, + "theoretical_loss": 4.288991586774211, + "tokens_seen": 226361344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047040120361083247, + "loss": 3.2327, + "theoretical_loss": 4.288839204200003, + "tokens_seen": 226426880 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703911735205617, + "loss": 2.9544, + "theoretical_loss": 4.288686878069666, + "tokens_seen": 226492416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703811434302909, + "loss": 3.1703, + "theoretical_loss": 4.288534608345969, + "tokens_seen": 226557952 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047037111334002007, + "loss": 3.1722, + "theoretical_loss": 4.288382394991717, + "tokens_seen": 226623488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047036108324974925, + "loss": 3.0562, + "theoretical_loss": 4.288230237969751, + "tokens_seen": 226689024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703510531594785, + "loss": 3.074, + "theoretical_loss": 4.288078137242946, + "tokens_seen": 226754560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703410230692076, + "loss": 2.9752, + "theoretical_loss": 4.287926092774212, + "tokens_seen": 226820096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047033099297893685, + "loss": 2.8926, + "theoretical_loss": 4.287774104526496, + "tokens_seen": 226885632 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470320962888666, + "loss": 3.2959, + "theoretical_loss": 4.287622172462777, + "tokens_seen": 226951168 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703109327983952, + "loss": 3.1867, + "theoretical_loss": 4.287470296546073, + "tokens_seen": 227016704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004703009027081244, + "loss": 3.0586, + "theoretical_loss": 4.287318476739433, + "tokens_seen": 227082240 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047029087261785357, + "loss": 3.1358, + "theoretical_loss": 4.287166713005945, + "tokens_seen": 227147776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047028084252758275, + "loss": 3.1048, + "theoretical_loss": 4.287015005308728, + "tokens_seen": 227213312 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047027081243731193, + "loss": 3.1529, + "theoretical_loss": 4.286863353610939, + "tokens_seen": 227278848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702607823470411, + "loss": 3.0266, + "theoretical_loss": 4.286711757875768, + "tokens_seen": 227344384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047025075225677035, + "loss": 3.1715, + "theoretical_loss": 4.28656021806644, + "tokens_seen": 227409920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702407221664995, + "loss": 3.0616, + "theoretical_loss": 4.286408734146216, + "tokens_seen": 227475456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702306920762287, + "loss": 3.1768, + "theoretical_loss": 4.28625730607839, + "tokens_seen": 227540992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047022066198595784, + "loss": 3.1572, + "theoretical_loss": 4.286105933826292, + "tokens_seen": 227606528 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004702106318956871, + "loss": 3.07, + "theoretical_loss": 4.285954617353285, + "tokens_seen": 227672064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 136262, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.876788377761841, + "objective/train/theoretical_loss": 4.28580335662277, + "objective/train/tokens_used": 248197600, + "theoretical_loss": 4.28580335662277, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047020060180541626, + "loss": 3.0746, + "theoretical_loss": 4.28580335662277, + "tokens_seen": 227737600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047019057171514544, + "loss": 3.2485, + "theoretical_loss": 4.285652151598178, + "tokens_seen": 227803136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701805416248746, + "loss": 3.2203, + "theoretical_loss": 4.285501002242977, + "tokens_seen": 227868672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047017051153460385, + "loss": 3.0961, + "theoretical_loss": 4.285349908520671, + "tokens_seen": 227934208 + }, + { + "epoch": 0.07, + "learning_rate": 0.000470160481444333, + "loss": 3.1869, + "theoretical_loss": 4.285198870394796, + "tokens_seen": 227999744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701504513540622, + "loss": 3.154, + "theoretical_loss": 4.28504788782892, + "tokens_seen": 228065280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047014042126379134, + "loss": 3.1927, + "theoretical_loss": 4.284896960786653, + "tokens_seen": 228130816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701303911735206, + "loss": 3.2737, + "theoretical_loss": 4.284746089231631, + "tokens_seen": 228196352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047012036108324976, + "loss": 3.299, + "theoretical_loss": 4.28459527312753, + "tokens_seen": 228261888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047011033099297894, + "loss": 2.9585, + "theoretical_loss": 4.284444512438057, + "tokens_seen": 228327424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004701003009027081, + "loss": 3.1119, + "theoretical_loss": 4.2842938071269545, + "tokens_seen": 228392960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700902708124373, + "loss": 3.165, + "theoretical_loss": 4.284143157157999, + "tokens_seen": 228458496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700802407221665, + "loss": 3.1586, + "theoretical_loss": 4.2839925624949995, + "tokens_seen": 228524032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700702106318957, + "loss": 3.1651, + "theoretical_loss": 4.283842023101803, + "tokens_seen": 228589568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047006018054162484, + "loss": 3.2229, + "theoretical_loss": 4.283691538942286, + "tokens_seen": 228655104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700501504513541, + "loss": 3.192, + "theoretical_loss": 4.28354110998036, + "tokens_seen": 228720640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047004012036108326, + "loss": 3.03, + "theoretical_loss": 4.283390736179975, + "tokens_seen": 228786176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047003009027081244, + "loss": 3.3334, + "theoretical_loss": 4.283240417505107, + "tokens_seen": 228851712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700200601805416, + "loss": 2.9563, + "theoretical_loss": 4.2830901539197725, + "tokens_seen": 228917248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004700100300902708, + "loss": 3.0645, + "theoretical_loss": 4.282939945388019, + "tokens_seen": 228982784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00047, + "loss": 3.2184, + "theoretical_loss": 4.2827897918739275, + "tokens_seen": 229048320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699899699097292, + "loss": 3.3057, + "theoretical_loss": 4.282639693341613, + "tokens_seen": 229113856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046997993981945835, + "loss": 3.179, + "theoretical_loss": 4.282489649755225, + "tokens_seen": 229179392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699699097291876, + "loss": 3.0861, + "theoretical_loss": 4.282339661078946, + "tokens_seen": 229244928 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046995987963891676, + "loss": 3.2011, + "theoretical_loss": 4.282189727276991, + "tokens_seen": 229310464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 137363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0329909324645996, + "objective/train/theoretical_loss": 4.282039848313611, + "objective/train/tokens_used": 249836000, + "theoretical_loss": 4.282039848313611, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046994984954864595, + "loss": 3.003, + "theoretical_loss": 4.282039848313611, + "tokens_seen": 229376000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699398194583752, + "loss": 3.2598, + "theoretical_loss": 4.281890024153089, + "tokens_seen": 229441536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004699297893681043, + "loss": 3.1051, + "theoretical_loss": 4.2817402547597405, + "tokens_seen": 229507072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046991975927783354, + "loss": 3.2666, + "theoretical_loss": 4.281590540097916, + "tokens_seen": 229572608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046990972918756267, + "loss": 3.1706, + "theoretical_loss": 4.281440880131999, + "tokens_seen": 229638144 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698996990972919, + "loss": 2.882, + "theoretical_loss": 4.2812912748264065, + "tokens_seen": 229703680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698896690070211, + "loss": 3.2516, + "theoretical_loss": 4.281141724145588, + "tokens_seen": 229769216 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046987963891675027, + "loss": 3.1911, + "theoretical_loss": 4.280992228054027, + "tokens_seen": 229834752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046986960882647945, + "loss": 3.337, + "theoretical_loss": 4.28084278651624, + "tokens_seen": 229900288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698595787362087, + "loss": 3.1908, + "theoretical_loss": 4.280693399496776, + "tokens_seen": 229965824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698495486459378, + "loss": 3.0165, + "theoretical_loss": 4.280544066960219, + "tokens_seen": 230031360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046983951855566705, + "loss": 2.9808, + "theoretical_loss": 4.280394788871183, + "tokens_seen": 230096896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698294884653962, + "loss": 3.0817, + "theoretical_loss": 4.280245565194319, + "tokens_seen": 230162432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698194583751254, + "loss": 3.208, + "theoretical_loss": 4.280096395894308, + "tokens_seen": 230227968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004698094282848546, + "loss": 3.1627, + "theoretical_loss": 4.279947280935864, + "tokens_seen": 230293504 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046979939819458377, + "loss": 3.0551, + "theoretical_loss": 4.279798220283736, + "tokens_seen": 230359040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046978936810431295, + "loss": 3.0795, + "theoretical_loss": 4.279649213902704, + "tokens_seen": 230424576 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046977933801404213, + "loss": 3.3708, + "theoretical_loss": 4.2795002617575815, + "tokens_seen": 230490112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697693079237713, + "loss": 3.1081, + "theoretical_loss": 4.279351363813215, + "tokens_seen": 230555648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046975927783350055, + "loss": 2.8502, + "theoretical_loss": 4.2792025200344845, + "tokens_seen": 230621184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697492477432297, + "loss": 3.1672, + "theoretical_loss": 4.279053730386301, + "tokens_seen": 230686720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697392176529589, + "loss": 3.1429, + "theoretical_loss": 4.278904994833609, + "tokens_seen": 230752256 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046972918756268804, + "loss": 3.0444, + "theoretical_loss": 4.278756313341386, + "tokens_seen": 230817792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004697191574724173, + "loss": 3.0223, + "theoretical_loss": 4.278607685874642, + "tokens_seen": 230883328 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046970912738214646, + "loss": 2.8679, + "theoretical_loss": 4.278459112398418, + "tokens_seen": 230948864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 138640, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.336690664291382, + "objective/train/theoretical_loss": 4.278310592877791, + "objective/train/tokens_used": 251474400, + "theoretical_loss": 4.278310592877791, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046969909729187564, + "loss": 2.9576, + "theoretical_loss": 4.278310592877791, + "tokens_seen": 231014400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696890672016048, + "loss": 2.9152, + "theoretical_loss": 4.2781621272778665, + "tokens_seen": 231079936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046967903711133405, + "loss": 3.0227, + "theoretical_loss": 4.278013715563786, + "tokens_seen": 231145472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696690070210632, + "loss": 3.1015, + "theoretical_loss": 4.27786535770072, + "tokens_seen": 231211008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696589769307924, + "loss": 3.1436, + "theoretical_loss": 4.277717053653875, + "tokens_seen": 231276544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046964894684052154, + "loss": 3.424, + "theoretical_loss": 4.277568803388486, + "tokens_seen": 231342080 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696389167502508, + "loss": 2.9966, + "theoretical_loss": 4.277420606869824, + "tokens_seen": 231407616 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046962888665997996, + "loss": 2.9781, + "theoretical_loss": 4.27727246406319, + "tokens_seen": 231473152 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046961885656970914, + "loss": 3.153, + "theoretical_loss": 4.277124374933917, + "tokens_seen": 231538688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004696088264794383, + "loss": 3.3026, + "theoretical_loss": 4.276976339447372, + "tokens_seen": 231604224 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695987963891675, + "loss": 3.1475, + "theoretical_loss": 4.276828357568952, + "tokens_seen": 231669760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695887662988967, + "loss": 2.9757, + "theoretical_loss": 4.276680429264088, + "tokens_seen": 231735296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695787362086259, + "loss": 3.2429, + "theoretical_loss": 4.276532554498242, + "tokens_seen": 231800832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046956870611835505, + "loss": 2.9495, + "theoretical_loss": 4.276384733236909, + "tokens_seen": 231866368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695586760280843, + "loss": 3.2042, + "theoretical_loss": 4.276236965445612, + "tokens_seen": 231931904 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046954864593781346, + "loss": 3.0181, + "theoretical_loss": 4.276089251089914, + "tokens_seen": 231997440 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046953861584754264, + "loss": 3.0194, + "theoretical_loss": 4.275941590135401, + "tokens_seen": 232062976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695285857572718, + "loss": 3.1368, + "theoretical_loss": 4.275793982547698, + "tokens_seen": 232128512 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469518555667001, + "loss": 3.1842, + "theoretical_loss": 4.275646428292456, + "tokens_seen": 232194048 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004695085255767302, + "loss": 3.0394, + "theoretical_loss": 4.275498927335362, + "tokens_seen": 232259584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694984954864594, + "loss": 3.2062, + "theoretical_loss": 4.275351479642134, + "tokens_seen": 232325120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046948846539618855, + "loss": 3.2058, + "theoretical_loss": 4.27520408517852, + "tokens_seen": 232390656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694784353059178, + "loss": 2.9021, + "theoretical_loss": 4.275056743910301, + "tokens_seen": 232456192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694684052156469, + "loss": 3.1015, + "theoretical_loss": 4.2749094558032885, + "tokens_seen": 232521728 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046945837512537615, + "loss": 3.1834, + "theoretical_loss": 4.274762220823328, + "tokens_seen": 232587264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 139202, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1069910526275635, + "objective/train/theoretical_loss": 4.274615038936293, + "objective/train/tokens_used": 253112800, + "theoretical_loss": 4.274615038936293, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694483450351053, + "loss": 3.0744, + "theoretical_loss": 4.274615038936293, + "tokens_seen": 232652800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694383149448345, + "loss": 3.0827, + "theoretical_loss": 4.274467910108093, + "tokens_seen": 232718336 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004694282848545637, + "loss": 3.0734, + "theoretical_loss": 4.274320834304666, + "tokens_seen": 232783872 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046941825476429287, + "loss": 3.1049, + "theoretical_loss": 4.27417381149198, + "tokens_seen": 232849408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046940822467402205, + "loss": 3.0458, + "theoretical_loss": 4.274026841636037, + "tokens_seen": 232914944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693981945837513, + "loss": 3.2492, + "theoretical_loss": 4.273879924702872, + "tokens_seen": 232980480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693881644934804, + "loss": 3.0141, + "theoretical_loss": 4.273733060658546, + "tokens_seen": 233046016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046937813440320965, + "loss": 2.9252, + "theoretical_loss": 4.273586249469155, + "tokens_seen": 233111552 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046936810431293883, + "loss": 3.0968, + "theoretical_loss": 4.273439491100828, + "tokens_seen": 233177088 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469358074222668, + "loss": 2.9294, + "theoretical_loss": 4.273292785519722, + "tokens_seen": 233242624 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693480441323972, + "loss": 2.9721, + "theoretical_loss": 4.273146132692023, + "tokens_seen": 233308160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693380140421264, + "loss": 3.2223, + "theoretical_loss": 4.272999532583955, + "tokens_seen": 233373696 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046932798395185555, + "loss": 3.1393, + "theoretical_loss": 4.272852985161768, + "tokens_seen": 233439232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693179538615848, + "loss": 3.2015, + "theoretical_loss": 4.272706490391743, + "tokens_seen": 233504768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004693079237713139, + "loss": 3.0477, + "theoretical_loss": 4.272560048240195, + "tokens_seen": 233570304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046929789368104315, + "loss": 3.0628, + "theoretical_loss": 4.272413658673468, + "tokens_seen": 233635840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692878635907723, + "loss": 3.2822, + "theoretical_loss": 4.272267321657937, + "tokens_seen": 233701376 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692778335005015, + "loss": 3.2682, + "theoretical_loss": 4.272121037160009, + "tokens_seen": 233766912 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692678034102307, + "loss": 3.1303, + "theoretical_loss": 4.271974805146121, + "tokens_seen": 233832448 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692577733199599, + "loss": 3.1491, + "theoretical_loss": 4.27182862558274, + "tokens_seen": 233897984 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046924774322968906, + "loss": 3.1131, + "theoretical_loss": 4.271682498436365, + "tokens_seen": 233963520 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046923771313941824, + "loss": 3.1811, + "theoretical_loss": 4.271536423673528, + "tokens_seen": 234029056 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004692276830491474, + "loss": 3.2045, + "theoretical_loss": 4.271390401260785, + "tokens_seen": 234094592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046921765295887666, + "loss": 3.1978, + "theoretical_loss": 4.271244431164731, + "tokens_seen": 234160128 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046920762286860584, + "loss": 3.2577, + "theoretical_loss": 4.271098513351986, + "tokens_seen": 234225664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 140513, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1093058586120605, + "objective/train/theoretical_loss": 4.270952647789202, + "objective/train/tokens_used": 254751200, + "theoretical_loss": 4.270952647789202, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469197592778335, + "loss": 3.3018, + "theoretical_loss": 4.270952647789202, + "tokens_seen": 234291200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046918756268806425, + "loss": 3.1679, + "theoretical_loss": 4.2708068344430625, + "tokens_seen": 234356736 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691775325977934, + "loss": 3.092, + "theoretical_loss": 4.270661073280282, + "tokens_seen": 234422272 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691675025075226, + "loss": 3.2188, + "theoretical_loss": 4.2705153642676015, + "tokens_seen": 234487808 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046915747241725174, + "loss": 3.162, + "theoretical_loss": 4.270369707371799, + "tokens_seen": 234553344 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469147442326981, + "loss": 3.123, + "theoretical_loss": 4.270224102559677, + "tokens_seen": 234618880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046913741223671016, + "loss": 3.2137, + "theoretical_loss": 4.270078549798071, + "tokens_seen": 234684416 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046912738214643934, + "loss": 3.1062, + "theoretical_loss": 4.269933049053847, + "tokens_seen": 234749952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691173520561685, + "loss": 3.2631, + "theoretical_loss": 4.269787600293903, + "tokens_seen": 234815488 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004691073219658977, + "loss": 3.0201, + "theoretical_loss": 4.269642203485162, + "tokens_seen": 234881024 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690972918756269, + "loss": 2.9759, + "theoretical_loss": 4.269496858594582, + "tokens_seen": 234946560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690872617853561, + "loss": 3.1686, + "theoretical_loss": 4.269351565589151, + "tokens_seen": 235012096 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046907723169508525, + "loss": 3.1832, + "theoretical_loss": 4.269206324435885, + "tokens_seen": 235077632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690672016048145, + "loss": 3.1976, + "theoretical_loss": 4.26906113510183, + "tokens_seen": 235143168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046905717151454366, + "loss": 3.1711, + "theoretical_loss": 4.268915997554065, + "tokens_seen": 235208704 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046904714142427284, + "loss": 3.105, + "theoretical_loss": 4.268770911759697, + "tokens_seen": 235274240 + }, + { + "epoch": 0.07, + "learning_rate": 0.000469037111334002, + "loss": 2.9095, + "theoretical_loss": 4.268625877685863, + "tokens_seen": 235339776 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690270812437312, + "loss": 3.0525, + "theoretical_loss": 4.268480895299731, + "tokens_seen": 235405312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690170511534604, + "loss": 3.2324, + "theoretical_loss": 4.268335964568497, + "tokens_seen": 235470848 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004690070210631896, + "loss": 3.1499, + "theoretical_loss": 4.26819108545939, + "tokens_seen": 235536384 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046899699097291875, + "loss": 3.1792, + "theoretical_loss": 4.268046257939666, + "tokens_seen": 235601920 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468986960882648, + "loss": 3.2761, + "theoretical_loss": 4.2679014819766135, + "tokens_seen": 235667456 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689769307923771, + "loss": 3.0686, + "theoretical_loss": 4.267756757537548, + "tokens_seen": 235732992 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046896690070210635, + "loss": 3.2238, + "theoretical_loss": 4.267612084589816, + "tokens_seen": 235798528 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046895687061183553, + "loss": 3.346, + "theoretical_loss": 4.267467463100796, + "tokens_seen": 235864064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 141147, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.236262321472168, + "objective/train/theoretical_loss": 4.267322893037893, + "objective/train/tokens_used": 256389600, + "theoretical_loss": 4.267322893037893, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689468405215647, + "loss": 3.064, + "theoretical_loss": 4.267322893037893, + "tokens_seen": 235929600 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689368104312939, + "loss": 3.0627, + "theoretical_loss": 4.267178374368543, + "tokens_seen": 235995136 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046892678034102307, + "loss": 3.1095, + "theoretical_loss": 4.267033907060211, + "tokens_seen": 236060672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046891675025075225, + "loss": 3.0446, + "theoretical_loss": 4.266889491080392, + "tokens_seen": 236126208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004689067201604815, + "loss": 3.1192, + "theoretical_loss": 4.2667451263966125, + "tokens_seen": 236191744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688966900702106, + "loss": 2.9901, + "theoretical_loss": 4.266600812976426, + "tokens_seen": 236257280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046888665997993985, + "loss": 3.1156, + "theoretical_loss": 4.2664565507874155, + "tokens_seen": 236322816 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046887662988966903, + "loss": 3.1053, + "theoretical_loss": 4.266312339797195, + "tokens_seen": 236388352 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688665997993982, + "loss": 3.0449, + "theoretical_loss": 4.266168179973407, + "tokens_seen": 236453888 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688565697091274, + "loss": 3.2738, + "theoretical_loss": 4.266024071283725, + "tokens_seen": 236519424 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688465396188566, + "loss": 2.8719, + "theoretical_loss": 4.265880013695849, + "tokens_seen": 236584960 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046883650952858575, + "loss": 3.2135, + "theoretical_loss": 4.265736007177511, + "tokens_seen": 236650496 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468826479438315, + "loss": 3.0794, + "theoretical_loss": 4.265592051696469, + "tokens_seen": 236716032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004688164493480441, + "loss": 2.9949, + "theoretical_loss": 4.265448147220515, + "tokens_seen": 236781568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046880641925777335, + "loss": 2.8953, + "theoretical_loss": 4.265304293717467, + "tokens_seen": 236847104 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687963891675025, + "loss": 2.9984, + "theoretical_loss": 4.2651604911551715, + "tokens_seen": 236912640 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687863590772317, + "loss": 3.054, + "theoretical_loss": 4.265016739501508, + "tokens_seen": 236978176 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687763289869609, + "loss": 3.052, + "theoretical_loss": 4.264873038724381, + "tokens_seen": 237043712 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687662988966901, + "loss": 3.0228, + "theoretical_loss": 4.264729388791726, + "tokens_seen": 237109248 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046875626880641926, + "loss": 3.0213, + "theoretical_loss": 4.264585789671508, + "tokens_seen": 237174784 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046874623871614844, + "loss": 3.2672, + "theoretical_loss": 4.26444224133172, + "tokens_seen": 237240320 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687362086258776, + "loss": 3.0529, + "theoretical_loss": 4.264298743740384, + "tokens_seen": 237305856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046872617853560686, + "loss": 2.9217, + "theoretical_loss": 4.264155296865552, + "tokens_seen": 237371392 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468716148445336, + "loss": 3.0414, + "theoretical_loss": 4.264011900675305, + "tokens_seen": 237436928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004687061183550652, + "loss": 3.0795, + "theoretical_loss": 4.26386855513775, + "tokens_seen": 237502464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 142360, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9394404888153076, + "objective/train/theoretical_loss": 4.263725260221027, + "objective/train/tokens_used": 258028000, + "theoretical_loss": 4.263725260221027, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686960882647944, + "loss": 3.1755, + "theoretical_loss": 4.263725260221027, + "tokens_seen": 237568000 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686860581745236, + "loss": 3.2614, + "theoretical_loss": 4.263582015893302, + "tokens_seen": 237633536 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046867602808425276, + "loss": 3.2817, + "theoretical_loss": 4.263438822122771, + "tokens_seen": 237699072 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046866599799398194, + "loss": 3.1451, + "theoretical_loss": 4.263295678877658, + "tokens_seen": 237764608 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686559679037111, + "loss": 3.2993, + "theoretical_loss": 4.263152586126217, + "tokens_seen": 237830144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046864593781344036, + "loss": 3.1979, + "theoretical_loss": 4.263009543836728, + "tokens_seen": 237895680 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686359077231695, + "loss": 3.1446, + "theoretical_loss": 4.2628665519775035, + "tokens_seen": 237961216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686258776328987, + "loss": 3.3129, + "theoretical_loss": 4.262723610516881, + "tokens_seen": 238026752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046861584754262785, + "loss": 3.0843, + "theoretical_loss": 4.26258071942323, + "tokens_seen": 238092288 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004686058174523571, + "loss": 3.0462, + "theoretical_loss": 4.262437878664945, + "tokens_seen": 238157824 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046859578736208626, + "loss": 3.1094, + "theoretical_loss": 4.262295088210451, + "tokens_seen": 238223360 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046858575727181545, + "loss": 3.0836, + "theoretical_loss": 4.262152348028202, + "tokens_seen": 238288896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685757271815446, + "loss": 3.2288, + "theoretical_loss": 4.26200965808668, + "tokens_seen": 238354432 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685656970912738, + "loss": 3.2874, + "theoretical_loss": 4.261867018354394, + "tokens_seen": 238419968 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468555667001003, + "loss": 3.0589, + "theoretical_loss": 4.261724428799884, + "tokens_seen": 238485504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685456369107322, + "loss": 3.0927, + "theoretical_loss": 4.261581889391715, + "tokens_seen": 238551040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046853560682046135, + "loss": 3.2002, + "theoretical_loss": 4.2614394000984825, + "tokens_seen": 238616576 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004685255767301906, + "loss": 3.1648, + "theoretical_loss": 4.261296960888811, + "tokens_seen": 238682112 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046851554663991977, + "loss": 3.3789, + "theoretical_loss": 4.2611545717313515, + "tokens_seen": 238747648 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046850551654964895, + "loss": 3.0458, + "theoretical_loss": 4.261012232594784, + "tokens_seen": 238813184 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046849548645937813, + "loss": 2.9991, + "theoretical_loss": 4.260869943447817, + "tokens_seen": 238878720 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684854563691073, + "loss": 3.0852, + "theoretical_loss": 4.260727704259187, + "tokens_seen": 238944256 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684754262788365, + "loss": 3.372, + "theoretical_loss": 4.260585514997658, + "tokens_seen": 239009792 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046846539618856573, + "loss": 3.0656, + "theoretical_loss": 4.260443375632022, + "tokens_seen": 239075328 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684553660982949, + "loss": 3.048, + "theoretical_loss": 4.260301286131099, + "tokens_seen": 239140864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 143562, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0513010025024414, + "objective/train/theoretical_loss": 4.26015924646374, + "objective/train/tokens_used": 259666400, + "theoretical_loss": 4.26015924646374, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684453360080241, + "loss": 2.9484, + "theoretical_loss": 4.26015924646374, + "tokens_seen": 239206400 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046843530591775327, + "loss": 3.4032, + "theoretical_loss": 4.26001725659882, + "tokens_seen": 239271936 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046842527582748245, + "loss": 3.1223, + "theoretical_loss": 4.2598753165052425, + "tokens_seen": 239337472 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684152457372117, + "loss": 3.1745, + "theoretical_loss": 4.259733426151941, + "tokens_seen": 239403008 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004684052156469408, + "loss": 3.0693, + "theoretical_loss": 4.259591585507876, + "tokens_seen": 239468544 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046839518555667005, + "loss": 3.2599, + "theoretical_loss": 4.259449794542036, + "tokens_seen": 239534080 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046838515546639923, + "loss": 3.051, + "theoretical_loss": 4.259308053223437, + "tokens_seen": 239599616 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683751253761284, + "loss": 3.2346, + "theoretical_loss": 4.259166361521121, + "tokens_seen": 239665152 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683650952858576, + "loss": 3.1637, + "theoretical_loss": 4.259024719404161, + "tokens_seen": 239730688 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683550651955868, + "loss": 3.0852, + "theoretical_loss": 4.258883126841656, + "tokens_seen": 239796224 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046834503510531595, + "loss": 3.0005, + "theoretical_loss": 4.258741583802734, + "tokens_seen": 239861760 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683350050150452, + "loss": 3.2479, + "theoretical_loss": 4.258600090256548, + "tokens_seen": 239927296 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683249749247743, + "loss": 2.7133, + "theoretical_loss": 4.258458646172281, + "tokens_seen": 239992832 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046831494483450355, + "loss": 3.1386, + "theoretical_loss": 4.258317251519143, + "tokens_seen": 240058368 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004683049147442327, + "loss": 3.2625, + "theoretical_loss": 4.258175906266371, + "tokens_seen": 240123904 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682948846539619, + "loss": 3.3216, + "theoretical_loss": 4.2580346103832305, + "tokens_seen": 240189440 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682848545636911, + "loss": 3.1082, + "theoretical_loss": 4.257893363839013, + "tokens_seen": 240254976 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682748244734203, + "loss": 2.8933, + "theoretical_loss": 4.257752166603041, + "tokens_seen": 240320512 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046826479438314946, + "loss": 3.1841, + "theoretical_loss": 4.257611018644658, + "tokens_seen": 240386048 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046825476429287864, + "loss": 2.9544, + "theoretical_loss": 4.257469919933241, + "tokens_seen": 240451584 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682447342026078, + "loss": 3.0391, + "theoretical_loss": 4.257328870438193, + "tokens_seen": 240517120 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046823470411233706, + "loss": 2.95, + "theoretical_loss": 4.257187870128942, + "tokens_seen": 240582656 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682246740220662, + "loss": 3.0321, + "theoretical_loss": 4.257046918974945, + "tokens_seen": 240648192 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682146439317954, + "loss": 3.1266, + "theoretical_loss": 4.2569060169456865, + "tokens_seen": 240713728 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004682046138415246, + "loss": 3.1535, + "theoretical_loss": 4.256765164010678, + "tokens_seen": 240779264 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 144045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4141147136688232, + "objective/train/theoretical_loss": 4.256624360139456, + "objective/train/tokens_used": 261304800, + "theoretical_loss": 4.256624360139456, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681945837512538, + "loss": 3.314, + "theoretical_loss": 4.256624360139456, + "tokens_seen": 240844800 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046818455366098296, + "loss": 3.0936, + "theoretical_loss": 4.25648360530159, + "tokens_seen": 240910336 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046817452357071214, + "loss": 2.9071, + "theoretical_loss": 4.25634289946667, + "tokens_seen": 240975872 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681644934804413, + "loss": 3.0465, + "theoretical_loss": 4.256202242604317, + "tokens_seen": 241041408 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046815446339017056, + "loss": 3.2584, + "theoretical_loss": 4.256061634684178, + "tokens_seen": 241106944 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681444332998997, + "loss": 3.0502, + "theoretical_loss": 4.255921075675927, + "tokens_seen": 241172480 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681344032096289, + "loss": 3.1567, + "theoretical_loss": 4.255780565549264, + "tokens_seen": 241238016 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046812437311935805, + "loss": 3.0096, + "theoretical_loss": 4.255640104273919, + "tokens_seen": 241303552 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004681143430290873, + "loss": 2.7541, + "theoretical_loss": 4.255499691819647, + "tokens_seen": 241369088 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046810431293881646, + "loss": 3.1162, + "theoretical_loss": 4.255359328156229, + "tokens_seen": 241434624 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046809428284854565, + "loss": 3.1105, + "theoretical_loss": 4.255219013253475, + "tokens_seen": 241500160 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680842527582748, + "loss": 3.1662, + "theoretical_loss": 4.255078747081219, + "tokens_seen": 241565696 + }, + { + "epoch": 0.07, + "learning_rate": 0.000468074222668004, + "loss": 3.0573, + "theoretical_loss": 4.254938529609326, + "tokens_seen": 241631232 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680641925777332, + "loss": 3.2336, + "theoretical_loss": 4.254798360807684, + "tokens_seen": 241696768 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680541624874624, + "loss": 3.1564, + "theoretical_loss": 4.254658240646211, + "tokens_seen": 241762304 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046804413239719155, + "loss": 3.0097, + "theoretical_loss": 4.254518169094847, + "tokens_seen": 241827840 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004680341023069208, + "loss": 3.0337, + "theoretical_loss": 4.254378146123563, + "tokens_seen": 241893376 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046802407221664997, + "loss": 2.9549, + "theoretical_loss": 4.254238171702356, + "tokens_seen": 241958912 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046801404212637915, + "loss": 2.976, + "theoretical_loss": 4.2540982458012495, + "tokens_seen": 242024448 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046800401203610833, + "loss": 3.1121, + "theoretical_loss": 4.253958368390292, + "tokens_seen": 242089984 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679939819458375, + "loss": 3.1283, + "theoretical_loss": 4.2538185394395605, + "tokens_seen": 242155520 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679839518555667, + "loss": 3.2828, + "theoretical_loss": 4.253678758919156, + "tokens_seen": 242221056 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046797392176529593, + "loss": 3.1396, + "theoretical_loss": 4.25353902679921, + "tokens_seen": 242286592 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046796389167502505, + "loss": 3.2791, + "theoretical_loss": 4.253399343049878, + "tokens_seen": 242352128 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679538615847543, + "loss": 3.0178, + "theoretical_loss": 4.253259707641341, + "tokens_seen": 242417664 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 145400, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.983632802963257, + "objective/train/theoretical_loss": 4.253120120543809, + "objective/train/tokens_used": 262943200, + "theoretical_loss": 4.253120120543809, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679438314944834, + "loss": 3.1131, + "theoretical_loss": 4.253120120543809, + "tokens_seen": 242483200 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046793380140421265, + "loss": 3.1157, + "theoretical_loss": 4.252980581727517, + "tokens_seen": 242548736 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046792377131394183, + "loss": 3.3695, + "theoretical_loss": 4.2528410911627255, + "tokens_seen": 242614272 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467913741223671, + "loss": 3.0132, + "theoretical_loss": 4.2527016488197225, + "tokens_seen": 242679808 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004679037111334002, + "loss": 3.2259, + "theoretical_loss": 4.252562254668824, + "tokens_seen": 242745344 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046789368104312943, + "loss": 3.3286, + "theoretical_loss": 4.2524229086803675, + "tokens_seen": 242810880 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046788365095285856, + "loss": 2.9194, + "theoretical_loss": 4.252283610824723, + "tokens_seen": 242876416 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678736208625878, + "loss": 3.1186, + "theoretical_loss": 4.25214436107228, + "tokens_seen": 242941952 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678635907723169, + "loss": 3.0458, + "theoretical_loss": 4.25200515939346, + "tokens_seen": 243007488 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046785356068204616, + "loss": 3.1178, + "theoretical_loss": 4.251866005758707, + "tokens_seen": 243073024 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046784353059177534, + "loss": 2.9132, + "theoretical_loss": 4.251726900138494, + "tokens_seen": 243138560 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678335005015045, + "loss": 3.1386, + "theoretical_loss": 4.251587842503317, + "tokens_seen": 243204096 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678234704112337, + "loss": 3.1516, + "theoretical_loss": 4.2514488328237, + "tokens_seen": 243269632 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004678134403209629, + "loss": 3.0271, + "theoretical_loss": 4.251309871070193, + "tokens_seen": 243335168 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046780341023069206, + "loss": 3.1534, + "theoretical_loss": 4.25117095721337, + "tokens_seen": 243400704 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677933801404213, + "loss": 3.2002, + "theoretical_loss": 4.251032091223834, + "tokens_seen": 243466240 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677833500501504, + "loss": 3.2106, + "theoretical_loss": 4.250893273072213, + "tokens_seen": 243531776 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046777331995987966, + "loss": 3.0027, + "theoretical_loss": 4.250754502729161, + "tokens_seen": 243597312 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677632898696088, + "loss": 3.0599, + "theoretical_loss": 4.250615780165355, + "tokens_seen": 243662848 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467753259779338, + "loss": 2.7333, + "theoretical_loss": 4.250477105351502, + "tokens_seen": 243728384 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677432296890672, + "loss": 3.1958, + "theoretical_loss": 4.250338478258333, + "tokens_seen": 243793920 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677331995987964, + "loss": 3.2165, + "theoretical_loss": 4.250199898856604, + "tokens_seen": 243859456 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046772316950852556, + "loss": 3.0671, + "theoretical_loss": 4.250061367117097, + "tokens_seen": 243924992 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004677131394182548, + "loss": 3.0396, + "theoretical_loss": 4.249922883010623, + "tokens_seen": 243990528 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467703109327984, + "loss": 3.0198, + "theoretical_loss": 4.249784446508014, + "tokens_seen": 244056064 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 146117, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.144657850265503, + "objective/train/theoretical_loss": 4.24964605758013, + "objective/train/tokens_used": 264581600, + "theoretical_loss": 4.24964605758013, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046769307923771316, + "loss": 3.141, + "theoretical_loss": 4.24964605758013, + "tokens_seen": 244121600 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046768304914744234, + "loss": 3.0374, + "theoretical_loss": 4.249507716197858, + "tokens_seen": 244187136 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676730190571715, + "loss": 2.9348, + "theoretical_loss": 4.249369422332107, + "tokens_seen": 244252672 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046766298896690076, + "loss": 3.2713, + "theoretical_loss": 4.249231175953814, + "tokens_seen": 244318208 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676529588766299, + "loss": 3.0096, + "theoretical_loss": 4.249092977033943, + "tokens_seen": 244383744 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676429287863591, + "loss": 3.0521, + "theoretical_loss": 4.248954825543479, + "tokens_seen": 244449280 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046763289869608825, + "loss": 3.049, + "theoretical_loss": 4.248816721453438, + "tokens_seen": 244514816 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004676228686058175, + "loss": 3.0529, + "theoretical_loss": 4.248678664734857, + "tokens_seen": 244580352 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046761283851554666, + "loss": 2.8181, + "theoretical_loss": 4.248540655358801, + "tokens_seen": 244645888 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046760280842527585, + "loss": 2.9386, + "theoretical_loss": 4.2484026932963594, + "tokens_seen": 244711424 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467592778335005, + "loss": 3.1262, + "theoretical_loss": 4.248264778518647, + "tokens_seen": 244776960 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675827482447342, + "loss": 3.0133, + "theoretical_loss": 4.248126910996804, + "tokens_seen": 244842496 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675727181544634, + "loss": 2.8972, + "theoretical_loss": 4.2479890907019975, + "tokens_seen": 244908032 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675626880641926, + "loss": 2.9783, + "theoretical_loss": 4.2478513176054165, + "tokens_seen": 244973568 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046755265797392175, + "loss": 3.045, + "theoretical_loss": 4.247713591678278, + "tokens_seen": 245039104 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467542627883651, + "loss": 2.8848, + "theoretical_loss": 4.247575912891824, + "tokens_seen": 245104640 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046753259779338017, + "loss": 3.0541, + "theoretical_loss": 4.247438281217321, + "tokens_seen": 245170176 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046752256770310935, + "loss": 3.0006, + "theoretical_loss": 4.247300696626062, + "tokens_seen": 245235712 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046751253761283853, + "loss": 3.1447, + "theoretical_loss": 4.247163159089362, + "tokens_seen": 245301248 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004675025075225677, + "loss": 3.204, + "theoretical_loss": 4.247025668578564, + "tokens_seen": 245366784 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674924774322969, + "loss": 3.2769, + "theoretical_loss": 4.246888225065036, + "tokens_seen": 245432320 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046748244734202613, + "loss": 2.7871, + "theoretical_loss": 4.24675082852017, + "tokens_seen": 245497856 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046747241725175525, + "loss": 2.8847, + "theoretical_loss": 4.246613478915384, + "tokens_seen": 245563392 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674623871614845, + "loss": 3.0322, + "theoretical_loss": 4.246476176222119, + "tokens_seen": 245628928 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674523570712136, + "loss": 3.3512, + "theoretical_loss": 4.246338920411843, + "tokens_seen": 245694464 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 147054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2560789585113525, + "objective/train/theoretical_loss": 4.24620171145605, + "objective/train/tokens_used": 266220000, + "theoretical_loss": 4.24620171145605, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046744232698094285, + "loss": 3.1654, + "theoretical_loss": 4.24620171145605, + "tokens_seen": 245760000 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046743229689067203, + "loss": 2.945, + "theoretical_loss": 4.246064549326255, + "tokens_seen": 245825536 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674222668004012, + "loss": 2.9211, + "theoretical_loss": 4.245927433994002, + "tokens_seen": 245891072 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004674122367101304, + "loss": 3.1739, + "theoretical_loss": 4.245790365430857, + "tokens_seen": 245956608 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046740220661985963, + "loss": 3.2878, + "theoretical_loss": 4.245653343608412, + "tokens_seen": 246022144 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046739217652958876, + "loss": 2.9528, + "theoretical_loss": 4.245516368498285, + "tokens_seen": 246087680 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467382146439318, + "loss": 3.1466, + "theoretical_loss": 4.245379440072116, + "tokens_seen": 246153216 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673721163490471, + "loss": 2.9806, + "theoretical_loss": 4.245242558301572, + "tokens_seen": 246218752 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046736208625877636, + "loss": 2.9725, + "theoretical_loss": 4.2451057231583444, + "tokens_seen": 246284288 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046735205616850554, + "loss": 3.115, + "theoretical_loss": 4.24496893461415, + "tokens_seen": 246349824 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673420260782347, + "loss": 3.1308, + "theoretical_loss": 4.244832192640725, + "tokens_seen": 246415360 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673319959879639, + "loss": 3.1996, + "theoretical_loss": 4.24469549720984, + "tokens_seen": 246480896 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673219658976931, + "loss": 3.2534, + "theoretical_loss": 4.244558848293281, + "tokens_seen": 246546432 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046731193580742226, + "loss": 2.9928, + "theoretical_loss": 4.244422245862863, + "tokens_seen": 246611968 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004673019057171515, + "loss": 3.2244, + "theoretical_loss": 4.244285689890425, + "tokens_seen": 246677504 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672918756268806, + "loss": 3.103, + "theoretical_loss": 4.244149180347831, + "tokens_seen": 246743040 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046728184553660986, + "loss": 3.0084, + "theoretical_loss": 4.244012717206969, + "tokens_seen": 246808576 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467271815446339, + "loss": 3.0494, + "theoretical_loss": 4.24387630043975, + "tokens_seen": 246874112 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672617853560682, + "loss": 3.2686, + "theoretical_loss": 4.243739930018112, + "tokens_seen": 246939648 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672517552657974, + "loss": 3.0173, + "theoretical_loss": 4.243603605914016, + "tokens_seen": 247005184 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672417251755266, + "loss": 3.2096, + "theoretical_loss": 4.243467328099448, + "tokens_seen": 247070720 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046723169508525576, + "loss": 3.3022, + "theoretical_loss": 4.2433310965464175, + "tokens_seen": 247136256 + }, + { + "epoch": 0.07, + "learning_rate": 0.000467221664994985, + "loss": 3.2016, + "theoretical_loss": 4.24319491122696, + "tokens_seen": 247201792 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004672116349047141, + "loss": 3.0706, + "theoretical_loss": 4.243058772113131, + "tokens_seen": 247267328 + }, + { + "epoch": 0.07, + "learning_rate": 0.00046720160481444336, + "loss": 3.0435, + "theoretical_loss": 4.242922679177017, + "tokens_seen": 247332864 + }, + { + "epoch": 0.07, + "objective/train/docs_used": 147484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0415401458740234, + "objective/train/theoretical_loss": 4.2427866323907235, + "objective/train/tokens_used": 267858400, + "theoretical_loss": 4.2427866323907235, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671915747241725, + "loss": 3.1482, + "theoretical_loss": 4.2427866323907235, + "tokens_seen": 247398400 + }, + { + "epoch": 0.07, + "learning_rate": 0.0004671815446339017, + "loss": 3.0442, + "theoretical_loss": 4.242650631726383, + "tokens_seen": 247463936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671715145436309, + "loss": 3.1308, + "theoretical_loss": 4.24251467715615, + "tokens_seen": 247529472 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671614844533601, + "loss": 3.0189, + "theoretical_loss": 4.2423787686522045, + "tokens_seen": 247595008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046715145436308927, + "loss": 3.2769, + "theoretical_loss": 4.2422429061867515, + "tokens_seen": 247660544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046714142427281845, + "loss": 3.1076, + "theoretical_loss": 4.242107089732018, + "tokens_seen": 247726080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046713139418254763, + "loss": 2.9247, + "theoretical_loss": 4.241971319260257, + "tokens_seen": 247791616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046712136409227686, + "loss": 3.0152, + "theoretical_loss": 4.241835594743743, + "tokens_seen": 247857152 + }, + { + "epoch": 0.08, + "learning_rate": 0.000467111334002006, + "loss": 3.1742, + "theoretical_loss": 4.241699916154779, + "tokens_seen": 247922688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004671013039117352, + "loss": 2.9643, + "theoretical_loss": 4.241564283465689, + "tokens_seen": 247988224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046709127382146435, + "loss": 3.1792, + "theoretical_loss": 4.241428696648818, + "tokens_seen": 248053760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670812437311936, + "loss": 2.9589, + "theoretical_loss": 4.241293155676542, + "tokens_seen": 248119296 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046707121364092277, + "loss": 2.9542, + "theoretical_loss": 4.241157660521255, + "tokens_seen": 248184832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046706118355065195, + "loss": 3.0795, + "theoretical_loss": 4.241022211155379, + "tokens_seen": 248250368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046705115346038113, + "loss": 2.8946, + "theoretical_loss": 4.240886807551356, + "tokens_seen": 248315904 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046704112337011037, + "loss": 3.1148, + "theoretical_loss": 4.240751449681655, + "tokens_seen": 248381440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670310932798395, + "loss": 3.2406, + "theoretical_loss": 4.240616137518767, + "tokens_seen": 248446976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046702106318956873, + "loss": 3.0567, + "theoretical_loss": 4.240480871035208, + "tokens_seen": 248512512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046701103309929786, + "loss": 3.0778, + "theoretical_loss": 4.240345650203517, + "tokens_seen": 248578048 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004670010030090271, + "loss": 3.1317, + "theoretical_loss": 4.240210474996257, + "tokens_seen": 248643584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669909729187563, + "loss": 2.8978, + "theoretical_loss": 4.240075345386014, + "tokens_seen": 248709120 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046698094282848545, + "loss": 3.1967, + "theoretical_loss": 4.239940261345399, + "tokens_seen": 248774656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046697091273821464, + "loss": 3.2585, + "theoretical_loss": 4.239805222847047, + "tokens_seen": 248840192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669608826479438, + "loss": 3.2483, + "theoretical_loss": 4.239670229863615, + "tokens_seen": 248905728 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046695085255767305, + "loss": 3.1135, + "theoretical_loss": 4.2395352823677825, + "tokens_seen": 248971264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 148704, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5283594131469727, + "objective/train/theoretical_loss": 4.239400380332256, + "objective/train/tokens_used": 269496800, + "theoretical_loss": 4.239400380332256, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046694082246740223, + "loss": 2.9646, + "theoretical_loss": 4.239400380332256, + "tokens_seen": 249036800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669307923771314, + "loss": 3.1003, + "theoretical_loss": 4.239265523729765, + "tokens_seen": 249102336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004669207622868606, + "loss": 3.0558, + "theoretical_loss": 4.239130712533059, + "tokens_seen": 249167872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046691073219658983, + "loss": 3.2011, + "theoretical_loss": 4.2389959467149145, + "tokens_seen": 249233408 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046690070210631896, + "loss": 3.1159, + "theoretical_loss": 4.238861226248131, + "tokens_seen": 249298944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668906720160482, + "loss": 2.9774, + "theoretical_loss": 4.238726551105531, + "tokens_seen": 249364480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668806419257773, + "loss": 2.8515, + "theoretical_loss": 4.2385919212599585, + "tokens_seen": 249430016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046687061183550656, + "loss": 3.0703, + "theoretical_loss": 4.238457336684284, + "tokens_seen": 249495552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046686058174523574, + "loss": 3.0331, + "theoretical_loss": 4.2383227973514, + "tokens_seen": 249561088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668505516549649, + "loss": 2.9647, + "theoretical_loss": 4.238188303234223, + "tokens_seen": 249626624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668405215646941, + "loss": 3.195, + "theoretical_loss": 4.23805385430569, + "tokens_seen": 249692160 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668304914744233, + "loss": 3.1549, + "theoretical_loss": 4.237919450538767, + "tokens_seen": 249757696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046682046138415246, + "loss": 3.257, + "theoretical_loss": 4.237785091906437, + "tokens_seen": 249823232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668104312938817, + "loss": 3.1791, + "theoretical_loss": 4.23765077838171, + "tokens_seen": 249888768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004668004012036108, + "loss": 3.0966, + "theoretical_loss": 4.237516509937617, + "tokens_seen": 249954304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046679037111334006, + "loss": 3.0634, + "theoretical_loss": 4.2373822865472155, + "tokens_seen": 250019840 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667803410230692, + "loss": 3.2033, + "theoretical_loss": 4.237248108183583, + "tokens_seen": 250085376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667703109327984, + "loss": 3.0081, + "theoretical_loss": 4.237113974819821, + "tokens_seen": 250150912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667602808425276, + "loss": 2.9963, + "theoretical_loss": 4.236979886429055, + "tokens_seen": 250216448 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667502507522568, + "loss": 3.0911, + "theoretical_loss": 4.236845842984433, + "tokens_seen": 250281984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046674022066198596, + "loss": 3.1609, + "theoretical_loss": 4.236711844459125, + "tokens_seen": 250347520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667301905717152, + "loss": 2.8868, + "theoretical_loss": 4.236577890826326, + "tokens_seen": 250413056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667201604814443, + "loss": 2.8415, + "theoretical_loss": 4.236443982059253, + "tokens_seen": 250478592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046671013039117356, + "loss": 3.1447, + "theoretical_loss": 4.236310118131144, + "tokens_seen": 250544128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004667001003009027, + "loss": 2.6377, + "theoretical_loss": 4.236176299015265, + "tokens_seen": 250609664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 149300, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2010979652404785, + "objective/train/theoretical_loss": 4.2360425246849, + "objective/train/tokens_used": 271135200, + "theoretical_loss": 4.2360425246849, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666900702106319, + "loss": 3.063, + "theoretical_loss": 4.2360425246849, + "tokens_seen": 250675200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666800401203611, + "loss": 3.1872, + "theoretical_loss": 4.2359087951133585, + "tokens_seen": 250740736 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666700100300903, + "loss": 3.1641, + "theoretical_loss": 4.235775110273972, + "tokens_seen": 250806272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046665997993981947, + "loss": 3.2843, + "theoretical_loss": 4.235641470140094, + "tokens_seen": 250871808 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046664994984954865, + "loss": 3.0174, + "theoretical_loss": 4.235507874685103, + "tokens_seen": 250937344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046663991975927783, + "loss": 2.8743, + "theoretical_loss": 4.235374323882399, + "tokens_seen": 251002880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046662988966900706, + "loss": 3.1391, + "theoretical_loss": 4.235240817705406, + "tokens_seen": 251068416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004666198595787362, + "loss": 3.0944, + "theoretical_loss": 4.235107356127567, + "tokens_seen": 251133952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046660982948846543, + "loss": 3.2513, + "theoretical_loss": 4.234973939122352, + "tokens_seen": 251199488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046659979939819455, + "loss": 2.703, + "theoretical_loss": 4.2348405666632525, + "tokens_seen": 251265024 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665897693079238, + "loss": 2.8943, + "theoretical_loss": 4.234707238723782, + "tokens_seen": 251330560 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046657973921765297, + "loss": 2.98, + "theoretical_loss": 4.234573955277476, + "tokens_seen": 251396096 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046656970912738215, + "loss": 3.1948, + "theoretical_loss": 4.234440716297895, + "tokens_seen": 251461632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046655967903711133, + "loss": 3.2021, + "theoretical_loss": 4.234307521758619, + "tokens_seen": 251527168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046654964894684057, + "loss": 3.0329, + "theoretical_loss": 4.234174371633253, + "tokens_seen": 251592704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665396188565697, + "loss": 3.0375, + "theoretical_loss": 4.234041265895425, + "tokens_seen": 251658240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046652958876629893, + "loss": 2.9757, + "theoretical_loss": 4.233908204518783, + "tokens_seen": 251723776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046651955867602806, + "loss": 2.956, + "theoretical_loss": 4.233775187476998, + "tokens_seen": 251789312 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004665095285857573, + "loss": 3.0873, + "theoretical_loss": 4.233642214743766, + "tokens_seen": 251854848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664994984954865, + "loss": 3.1756, + "theoretical_loss": 4.2335092862928025, + "tokens_seen": 251920384 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046648946840521565, + "loss": 3.047, + "theoretical_loss": 4.233376402097846, + "tokens_seen": 251985920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046647943831494484, + "loss": 2.7, + "theoretical_loss": 4.233243562132661, + "tokens_seen": 252051456 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466469408224674, + "loss": 2.8238, + "theoretical_loss": 4.233110766371029, + "tokens_seen": 252116992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664593781344032, + "loss": 3.0103, + "theoretical_loss": 4.232978014786756, + "tokens_seen": 252182528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046644934804413243, + "loss": 2.9662, + "theoretical_loss": 4.2328453073536725, + "tokens_seen": 252248064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 150610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.978238582611084, + "objective/train/theoretical_loss": 4.232712644045627, + "objective/train/tokens_used": 272773600, + "theoretical_loss": 4.232712644045627, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046643931795386156, + "loss": 3.1531, + "theoretical_loss": 4.232712644045627, + "tokens_seen": 252313600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664292878635908, + "loss": 3.2135, + "theoretical_loss": 4.232580024836494, + "tokens_seen": 252379136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004664192577733199, + "loss": 3.0233, + "theoretical_loss": 4.232447449700169, + "tokens_seen": 252444672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046640922768304916, + "loss": 2.8671, + "theoretical_loss": 4.23231491861057, + "tokens_seen": 252510208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046639919759277834, + "loss": 3.0974, + "theoretical_loss": 4.232182431541636, + "tokens_seen": 252575744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663891675025075, + "loss": 3.0586, + "theoretical_loss": 4.232049988467328, + "tokens_seen": 252641280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663791374122367, + "loss": 2.9412, + "theoretical_loss": 4.2319175893616325, + "tokens_seen": 252706816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046636910732196594, + "loss": 2.9511, + "theoretical_loss": 4.231785234198555, + "tokens_seen": 252772352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046635907723169506, + "loss": 2.9213, + "theoretical_loss": 4.231652922952122, + "tokens_seen": 252837888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663490471414243, + "loss": 2.9723, + "theoretical_loss": 4.231520655596388, + "tokens_seen": 252903424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004663390170511534, + "loss": 3.0252, + "theoretical_loss": 4.231388432105421, + "tokens_seen": 252968960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046632898696088266, + "loss": 3.1199, + "theoretical_loss": 4.231256252453317, + "tokens_seen": 253034496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046631895687061184, + "loss": 3.194, + "theoretical_loss": 4.231124116614194, + "tokens_seen": 253100032 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466308926780341, + "loss": 3.1208, + "theoretical_loss": 4.230992024562189, + "tokens_seen": 253165568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662988966900702, + "loss": 2.9426, + "theoretical_loss": 4.230859976271464, + "tokens_seen": 253231104 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662888665997994, + "loss": 3.0458, + "theoretical_loss": 4.2307279717162, + "tokens_seen": 253296640 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046627883650952857, + "loss": 3.193, + "theoretical_loss": 4.2305960108706016, + "tokens_seen": 253362176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662688064192578, + "loss": 3.1534, + "theoretical_loss": 4.230464093708894, + "tokens_seen": 253427712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046625877632898693, + "loss": 3.1943, + "theoretical_loss": 4.230332220205328, + "tokens_seen": 253493248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046624874623871616, + "loss": 3.0607, + "theoretical_loss": 4.230200390334172, + "tokens_seen": 253558784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046623871614844535, + "loss": 2.8528, + "theoretical_loss": 4.230068604069717, + "tokens_seen": 253624320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662286860581745, + "loss": 2.9088, + "theoretical_loss": 4.2299368613862764, + "tokens_seen": 253689856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662186559679037, + "loss": 3.2573, + "theoretical_loss": 4.229805162258187, + "tokens_seen": 253755392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004662086258776329, + "loss": 2.9836, + "theoretical_loss": 4.229673506659805, + "tokens_seen": 253820928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661985957873621, + "loss": 3.1131, + "theoretical_loss": 4.229541894565508, + "tokens_seen": 253886464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 151678, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8198091983795166, + "objective/train/theoretical_loss": 4.229410325949698, + "objective/train/tokens_used": 274412000, + "theoretical_loss": 4.229410325949698, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661885656970913, + "loss": 2.9257, + "theoretical_loss": 4.229410325949698, + "tokens_seen": 253952000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661785356068205, + "loss": 3.1713, + "theoretical_loss": 4.229278800786796, + "tokens_seen": 254017536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046616850551654967, + "loss": 2.9401, + "theoretical_loss": 4.229147319051247, + "tokens_seen": 254083072 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046615847542627885, + "loss": 2.8864, + "theoretical_loss": 4.2290158807175136, + "tokens_seen": 254148608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046614844533600803, + "loss": 3.1424, + "theoretical_loss": 4.228884485760086, + "tokens_seen": 254214144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046613841524573727, + "loss": 3.2015, + "theoretical_loss": 4.22875313415347, + "tokens_seen": 254279680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004661283851554664, + "loss": 3.1367, + "theoretical_loss": 4.228621825872198, + "tokens_seen": 254345216 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046611835506519563, + "loss": 2.9269, + "theoretical_loss": 4.228490560890819, + "tokens_seen": 254410752 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046610832497492475, + "loss": 3.1191, + "theoretical_loss": 4.228359339183908, + "tokens_seen": 254476288 + }, + { + "epoch": 0.08, + "learning_rate": 0.000466098294884654, + "loss": 3.2313, + "theoretical_loss": 4.228228160726059, + "tokens_seen": 254541824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046608826479438317, + "loss": 3.2163, + "theoretical_loss": 4.228097025491887, + "tokens_seen": 254607360 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046607823470411235, + "loss": 3.0566, + "theoretical_loss": 4.22796593345603, + "tokens_seen": 254672896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046606820461384153, + "loss": 2.9742, + "theoretical_loss": 4.227834884593148, + "tokens_seen": 254738432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046605817452357077, + "loss": 2.6936, + "theoretical_loss": 4.22770387887792, + "tokens_seen": 254803968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660481444332999, + "loss": 2.989, + "theoretical_loss": 4.227572916285046, + "tokens_seen": 254869504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046603811434302913, + "loss": 3.1684, + "theoretical_loss": 4.227441996789253, + "tokens_seen": 254935040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046602808425275826, + "loss": 3.0604, + "theoretical_loss": 4.227311120365281, + "tokens_seen": 255000576 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660180541624875, + "loss": 3.3224, + "theoretical_loss": 4.227180286987897, + "tokens_seen": 255066112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004660080240722167, + "loss": 3.2097, + "theoretical_loss": 4.227049496631889, + "tokens_seen": 255131648 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046599799398194586, + "loss": 3.1686, + "theoretical_loss": 4.226918749272063, + "tokens_seen": 255197184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046598796389167504, + "loss": 3.0575, + "theoretical_loss": 4.226788044883248, + "tokens_seen": 255262720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659779338014042, + "loss": 3.1225, + "theoretical_loss": 4.226657383440297, + "tokens_seen": 255328256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659679037111334, + "loss": 2.9431, + "theoretical_loss": 4.226526764918079, + "tokens_seen": 255393792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046595787362086263, + "loss": 3.1373, + "theoretical_loss": 4.2263961892914885, + "tokens_seen": 255459328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046594784353059176, + "loss": 3.2497, + "theoretical_loss": 4.226265656535437, + "tokens_seen": 255524864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 152377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.094141960144043, + "objective/train/theoretical_loss": 4.226135166624862, + "objective/train/tokens_used": 276050400, + "theoretical_loss": 4.226135166624862, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465937813440321, + "loss": 3.2235, + "theoretical_loss": 4.226135166624862, + "tokens_seen": 255590400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004659277833500501, + "loss": 3.158, + "theoretical_loss": 4.226004719534718, + "tokens_seen": 255655936 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046591775325977936, + "loss": 2.9932, + "theoretical_loss": 4.225874315239982, + "tokens_seen": 255721472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046590772316950854, + "loss": 2.9383, + "theoretical_loss": 4.225743953715653, + "tokens_seen": 255787008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658976930792377, + "loss": 3.0384, + "theoretical_loss": 4.225613634936749, + "tokens_seen": 255852544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658876629889669, + "loss": 3.1105, + "theoretical_loss": 4.22548335887831, + "tokens_seen": 255918080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046587763289869614, + "loss": 3.0846, + "theoretical_loss": 4.225353125515398, + "tokens_seen": 255983616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046586760280842526, + "loss": 3.013, + "theoretical_loss": 4.225222934823094, + "tokens_seen": 256049152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658575727181545, + "loss": 3.1509, + "theoretical_loss": 4.225092786776502, + "tokens_seen": 256114688 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658475426278836, + "loss": 2.9779, + "theoretical_loss": 4.224962681350746, + "tokens_seen": 256180224 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046583751253761286, + "loss": 2.778, + "theoretical_loss": 4.224832618520969, + "tokens_seen": 256245760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046582748244734204, + "loss": 3.263, + "theoretical_loss": 4.224702598262336, + "tokens_seen": 256311296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658174523570712, + "loss": 2.9598, + "theoretical_loss": 4.2245726205500365, + "tokens_seen": 256376832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004658074222668004, + "loss": 3.1954, + "theoretical_loss": 4.224442685359275, + "tokens_seen": 256442368 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657973921765296, + "loss": 2.9688, + "theoretical_loss": 4.22431279266528, + "tokens_seen": 256507904 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046578736208625877, + "loss": 3.1653, + "theoretical_loss": 4.2241829424433, + "tokens_seen": 256573440 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465777331995988, + "loss": 3.0042, + "theoretical_loss": 4.224053134668605, + "tokens_seen": 256638976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046576730190571713, + "loss": 2.9824, + "theoretical_loss": 4.223923369316485, + "tokens_seen": 256704512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046575727181544636, + "loss": 2.9607, + "theoretical_loss": 4.2237936463622505, + "tokens_seen": 256770048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046574724172517555, + "loss": 3.0764, + "theoretical_loss": 4.223663965781233, + "tokens_seen": 256835584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657372116349047, + "loss": 3.1061, + "theoretical_loss": 4.2235343275487836, + "tokens_seen": 256901120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657271815446339, + "loss": 2.9893, + "theoretical_loss": 4.223404731640277, + "tokens_seen": 256966656 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004657171514543631, + "loss": 2.9472, + "theoretical_loss": 4.223275178031105, + "tokens_seen": 257032192 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046570712136409227, + "loss": 3.0962, + "theoretical_loss": 4.223145666696682, + "tokens_seen": 257097728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656970912738215, + "loss": 3.2783, + "theoretical_loss": 4.223016197612442, + "tokens_seen": 257163264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 153004, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8444950580596924, + "objective/train/theoretical_loss": 4.222886770753841, + "objective/train/tokens_used": 277688800, + "theoretical_loss": 4.222886770753841, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046568706118355063, + "loss": 2.8758, + "theoretical_loss": 4.222886770753841, + "tokens_seen": 257228800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046567703109327987, + "loss": 3.2012, + "theoretical_loss": 4.222757386096354, + "tokens_seen": 257294336 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465667001003009, + "loss": 3.0649, + "theoretical_loss": 4.222628043615476, + "tokens_seen": 257359872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046565697091273823, + "loss": 3.1071, + "theoretical_loss": 4.2224987432867245, + "tokens_seen": 257425408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656469408224674, + "loss": 3.1137, + "theoretical_loss": 4.222369485085636, + "tokens_seen": 257490944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656369107321966, + "loss": 3.1091, + "theoretical_loss": 4.222240268987768, + "tokens_seen": 257556480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004656268806419258, + "loss": 2.8566, + "theoretical_loss": 4.222111094968698, + "tokens_seen": 257622016 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046561685055165495, + "loss": 3.0556, + "theoretical_loss": 4.2219819630040245, + "tokens_seen": 257687552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046560682046138414, + "loss": 3.1304, + "theoretical_loss": 4.2218528730693645, + "tokens_seen": 257753088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046559679037111337, + "loss": 2.937, + "theoretical_loss": 4.221723825140359, + "tokens_seen": 257818624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655867602808425, + "loss": 2.9502, + "theoretical_loss": 4.221594819192665, + "tokens_seen": 257884160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046557673019057173, + "loss": 3.1884, + "theoretical_loss": 4.221465855201963, + "tokens_seen": 257949696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655667001003009, + "loss": 3.1707, + "theoretical_loss": 4.221336933143952, + "tokens_seen": 258015232 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655566700100301, + "loss": 2.9765, + "theoretical_loss": 4.221208052994352, + "tokens_seen": 258080768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655466399197593, + "loss": 3.1372, + "theoretical_loss": 4.221079214728903, + "tokens_seen": 258146304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046553660982948846, + "loss": 3.0284, + "theoretical_loss": 4.2209504183233655, + "tokens_seen": 258211840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046552657973921764, + "loss": 3.1995, + "theoretical_loss": 4.22082166375352, + "tokens_seen": 258277376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004655165496489469, + "loss": 2.8978, + "theoretical_loss": 4.220692950995167, + "tokens_seen": 258342912 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465506519558676, + "loss": 3.0251, + "theoretical_loss": 4.220564280024126, + "tokens_seen": 258408448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046549648946840524, + "loss": 2.9952, + "theoretical_loss": 4.22043565081624, + "tokens_seen": 258473984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046548645937813436, + "loss": 3.1847, + "theoretical_loss": 4.220307063347369, + "tokens_seen": 258539520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654764292878636, + "loss": 2.9241, + "theoretical_loss": 4.220178517593393, + "tokens_seen": 258605056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654663991975928, + "loss": 2.8651, + "theoretical_loss": 4.220050013530215, + "tokens_seen": 258670592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046545636910732196, + "loss": 2.9641, + "theoretical_loss": 4.219921551133754, + "tokens_seen": 258736128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654463390170512, + "loss": 3.1291, + "theoretical_loss": 4.219793130379952, + "tokens_seen": 258801664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 154245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.155519485473633, + "objective/train/theoretical_loss": 4.21966475124477, + "objective/train/tokens_used": 279327200, + "theoretical_loss": 4.21966475124477, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654363089267803, + "loss": 3.232, + "theoretical_loss": 4.21966475124477, + "tokens_seen": 258867200 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046542627883650956, + "loss": 3.1265, + "theoretical_loss": 4.21953641370419, + "tokens_seen": 258932736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046541624874623874, + "loss": 2.8509, + "theoretical_loss": 4.21940811773421, + "tokens_seen": 258998272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004654062186559679, + "loss": 3.0692, + "theoretical_loss": 4.219279863310855, + "tokens_seen": 259063808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653961885656971, + "loss": 2.9505, + "theoretical_loss": 4.219151650410162, + "tokens_seen": 259129344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046538615847542634, + "loss": 3.049, + "theoretical_loss": 4.219023479008193, + "tokens_seen": 259194880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046537612838515546, + "loss": 3.0422, + "theoretical_loss": 4.21889534908103, + "tokens_seen": 259260416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653660982948847, + "loss": 3.2427, + "theoretical_loss": 4.218767260604771, + "tokens_seen": 259325952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653560682046138, + "loss": 2.8318, + "theoretical_loss": 4.218639213555537, + "tokens_seen": 259391488 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046534603811434306, + "loss": 2.817, + "theoretical_loss": 4.218511207909467, + "tokens_seen": 259457024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046533600802407224, + "loss": 2.9972, + "theoretical_loss": 4.218383243642723, + "tokens_seen": 259522560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653259779338014, + "loss": 2.8747, + "theoretical_loss": 4.2182553207314815, + "tokens_seen": 259588096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653159478435306, + "loss": 3.0103, + "theoretical_loss": 4.218127439151944, + "tokens_seen": 259653632 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004653059177532598, + "loss": 3.1678, + "theoretical_loss": 4.217999598880327, + "tokens_seen": 259719168 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046529588766298897, + "loss": 3.1776, + "theoretical_loss": 4.2178717998928725, + "tokens_seen": 259784704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652858575727182, + "loss": 3.0578, + "theoretical_loss": 4.217744042165835, + "tokens_seen": 259850240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046527582748244733, + "loss": 3.0386, + "theoretical_loss": 4.217616325675493, + "tokens_seen": 259915776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046526579739217656, + "loss": 2.9572, + "theoretical_loss": 4.2174886503981455, + "tokens_seen": 259981312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046525576730190575, + "loss": 2.985, + "theoretical_loss": 4.217361016310109, + "tokens_seen": 260046848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652457372116349, + "loss": 3.0667, + "theoretical_loss": 4.217233423387719, + "tokens_seen": 260112384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652357071213641, + "loss": 3.0605, + "theoretical_loss": 4.217105871607333, + "tokens_seen": 260177920 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652256770310933, + "loss": 3.354, + "theoretical_loss": 4.216978360945325, + "tokens_seen": 260243456 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046521564694082247, + "loss": 2.9401, + "theoretical_loss": 4.216850891378092, + "tokens_seen": 260308992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004652056168505517, + "loss": 3.0811, + "theoretical_loss": 4.216723462882047, + "tokens_seen": 260374528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046519558676028083, + "loss": 3.1081, + "theoretical_loss": 4.2165960754336265, + "tokens_seen": 260440064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 155441, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7761213779449463, + "objective/train/theoretical_loss": 4.216468729009281, + "objective/train/tokens_used": 280965600, + "theoretical_loss": 4.216468729009281, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046518555667001007, + "loss": 2.6967, + "theoretical_loss": 4.216468729009281, + "tokens_seen": 260505600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651755265797392, + "loss": 3.038, + "theoretical_loss": 4.216341423585486, + "tokens_seen": 260571136 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046516549648946843, + "loss": 3.3396, + "theoretical_loss": 4.216214159138732, + "tokens_seen": 260636672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651554663991976, + "loss": 3.0875, + "theoretical_loss": 4.216086935645532, + "tokens_seen": 260702208 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004651454363089268, + "loss": 3.1269, + "theoretical_loss": 4.2159597530824175, + "tokens_seen": 260767744 + }, + { + "epoch": 0.08, + "learning_rate": 0.000465135406218656, + "loss": 2.9654, + "theoretical_loss": 4.215832611425937, + "tokens_seen": 260833280 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046512537612838515, + "loss": 2.9985, + "theoretical_loss": 4.215705510652662, + "tokens_seen": 260898816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046511534603811434, + "loss": 3.0659, + "theoretical_loss": 4.215578450739182, + "tokens_seen": 260964352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046510531594784357, + "loss": 2.859, + "theoretical_loss": 4.215451431662104, + "tokens_seen": 261029888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650952858575727, + "loss": 3.0105, + "theoretical_loss": 4.215324453398057, + "tokens_seen": 261095424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046508525576730193, + "loss": 3.0586, + "theoretical_loss": 4.215197515923687, + "tokens_seen": 261160960 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650752256770311, + "loss": 3.0384, + "theoretical_loss": 4.21507061921566, + "tokens_seen": 261226496 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650651955867603, + "loss": 2.9461, + "theoretical_loss": 4.214943763250662, + "tokens_seen": 261292032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650551654964895, + "loss": 3.0812, + "theoretical_loss": 4.214816948005396, + "tokens_seen": 261357568 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046504513540621866, + "loss": 2.9739, + "theoretical_loss": 4.214690173456588, + "tokens_seen": 261423104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046503510531594784, + "loss": 3.2378, + "theoretical_loss": 4.214563439580979, + "tokens_seen": 261488640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650250752256771, + "loss": 2.9873, + "theoretical_loss": 4.214436746355331, + "tokens_seen": 261554176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004650150451354062, + "loss": 3.0108, + "theoretical_loss": 4.214310093756427, + "tokens_seen": 261619712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046500501504513544, + "loss": 2.9422, + "theoretical_loss": 4.214183481761064, + "tokens_seen": 261685248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046499498495486456, + "loss": 3.1189, + "theoretical_loss": 4.214056910346063, + "tokens_seen": 261750784 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649849548645938, + "loss": 3.1657, + "theoretical_loss": 4.2139303794882625, + "tokens_seen": 261816320 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464974924774323, + "loss": 2.7213, + "theoretical_loss": 4.213803889164518, + "tokens_seen": 261881856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046496489468405216, + "loss": 2.8726, + "theoretical_loss": 4.213677439351708, + "tokens_seen": 261947392 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046495486459378134, + "loss": 2.7336, + "theoretical_loss": 4.213551030026725, + "tokens_seen": 262012928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649448345035105, + "loss": 2.9864, + "theoretical_loss": 4.213424661166485, + "tokens_seen": 262078464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 156138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.797309398651123, + "objective/train/theoretical_loss": 4.21329833274792, + "objective/train/tokens_used": 282604000, + "theoretical_loss": 4.21329833274792, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649348044132397, + "loss": 2.5665, + "theoretical_loss": 4.21329833274792, + "tokens_seen": 262144000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046492477432296894, + "loss": 3.0384, + "theoretical_loss": 4.213172044747982, + "tokens_seen": 262209536 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046491474423269807, + "loss": 2.8508, + "theoretical_loss": 4.213045797143643, + "tokens_seen": 262275072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004649047141424273, + "loss": 2.933, + "theoretical_loss": 4.212919589911891, + "tokens_seen": 262340608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648946840521565, + "loss": 2.8909, + "theoretical_loss": 4.212793423029735, + "tokens_seen": 262406144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046488465396188566, + "loss": 3.0675, + "theoretical_loss": 4.212667296474204, + "tokens_seen": 262471680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046487462387161485, + "loss": 3.0812, + "theoretical_loss": 4.2125412102223425, + "tokens_seen": 262537216 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464864593781344, + "loss": 3.2516, + "theoretical_loss": 4.212415164251215, + "tokens_seen": 262602752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648545636910732, + "loss": 2.9727, + "theoretical_loss": 4.212289158537908, + "tokens_seen": 262668288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046484453360080244, + "loss": 3.2213, + "theoretical_loss": 4.212163193059521, + "tokens_seen": 262733824 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046483450351053157, + "loss": 3.0169, + "theoretical_loss": 4.2120372677931766, + "tokens_seen": 262799360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004648244734202608, + "loss": 3.2831, + "theoretical_loss": 4.211911382716014, + "tokens_seen": 262864896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046481444332998993, + "loss": 2.9297, + "theoretical_loss": 4.211785537805193, + "tokens_seen": 262930432 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046480441323971917, + "loss": 3.0726, + "theoretical_loss": 4.211659733037891, + "tokens_seen": 262995968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046479438314944835, + "loss": 3.0694, + "theoretical_loss": 4.2115339683913025, + "tokens_seen": 263061504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046478435305917753, + "loss": 2.8509, + "theoretical_loss": 4.211408243842643, + "tokens_seen": 263127040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647743229689067, + "loss": 3.0094, + "theoretical_loss": 4.2112825593691445, + "tokens_seen": 263192576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046476429287863595, + "loss": 3.1927, + "theoretical_loss": 4.21115691494806, + "tokens_seen": 263258112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647542627883651, + "loss": 3.0788, + "theoretical_loss": 4.211031310556661, + "tokens_seen": 263323648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647442326980943, + "loss": 2.9797, + "theoretical_loss": 4.210905746172235, + "tokens_seen": 263389184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046473420260782344, + "loss": 2.7703, + "theoretical_loss": 4.210780221772088, + "tokens_seen": 263454720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046472417251755267, + "loss": 2.9913, + "theoretical_loss": 4.210654737333549, + "tokens_seen": 263520256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004647141424272819, + "loss": 3.0574, + "theoretical_loss": 4.21052929283396, + "tokens_seen": 263585792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046470411233701103, + "loss": 2.9517, + "theoretical_loss": 4.210403888250685, + "tokens_seen": 263651328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046469408224674027, + "loss": 2.9469, + "theoretical_loss": 4.210278523561106, + "tokens_seen": 263716864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 157246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.921288251876831, + "objective/train/theoretical_loss": 4.2101531987426215, + "objective/train/tokens_used": 284242400, + "theoretical_loss": 4.2101531987426215, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646840521564694, + "loss": 3.1574, + "theoretical_loss": 4.2101531987426215, + "tokens_seen": 263782400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046467402206619863, + "loss": 2.9865, + "theoretical_loss": 4.210027913772651, + "tokens_seen": 263847936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646639919759278, + "loss": 2.9807, + "theoretical_loss": 4.209902668628629, + "tokens_seen": 263913472 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464653961885657, + "loss": 3.0513, + "theoretical_loss": 4.209777463288012, + "tokens_seen": 263979008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646439317953862, + "loss": 2.8659, + "theoretical_loss": 4.209652297728274, + "tokens_seen": 264044544 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046463390170511535, + "loss": 3.1063, + "theoretical_loss": 4.209527171926906, + "tokens_seen": 264110080 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046462387161484454, + "loss": 2.9362, + "theoretical_loss": 4.209402085861418, + "tokens_seen": 264175616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046461384152457377, + "loss": 3.1464, + "theoretical_loss": 4.209277039509338, + "tokens_seen": 264241152 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004646038114343029, + "loss": 3.0352, + "theoretical_loss": 4.209152032848213, + "tokens_seen": 264306688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046459378134403213, + "loss": 2.7659, + "theoretical_loss": 4.209027065855607, + "tokens_seen": 264372224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645837512537613, + "loss": 2.8415, + "theoretical_loss": 4.208902138509105, + "tokens_seen": 264437760 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645737211634905, + "loss": 3.2915, + "theoretical_loss": 4.208777250786305, + "tokens_seen": 264503296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645636910732197, + "loss": 3.0093, + "theoretical_loss": 4.20865240266483, + "tokens_seen": 264568832 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046455366098294886, + "loss": 3.0984, + "theoretical_loss": 4.208527594122316, + "tokens_seen": 264634368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046454363089267804, + "loss": 3.171, + "theoretical_loss": 4.208402825136419, + "tokens_seen": 264699904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645336008024073, + "loss": 3.2183, + "theoretical_loss": 4.208278095684813, + "tokens_seen": 264765440 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004645235707121364, + "loss": 3.1366, + "theoretical_loss": 4.20815340574519, + "tokens_seen": 264830976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046451354062186564, + "loss": 2.9705, + "theoretical_loss": 4.20802875529526, + "tokens_seen": 264896512 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046450351053159476, + "loss": 3.2497, + "theoretical_loss": 4.207904144312752, + "tokens_seen": 264962048 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464493480441324, + "loss": 3.1657, + "theoretical_loss": 4.2077795727754115, + "tokens_seen": 265027584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644834503510532, + "loss": 2.869, + "theoretical_loss": 4.207655040661004, + "tokens_seen": 265093120 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046447342026078236, + "loss": 2.8442, + "theoretical_loss": 4.207530547947311, + "tokens_seen": 265158656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046446339017051154, + "loss": 3.1521, + "theoretical_loss": 4.207406094612133, + "tokens_seen": 265224192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644533600802407, + "loss": 2.845, + "theoretical_loss": 4.207281680633289, + "tokens_seen": 265289728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644433299899699, + "loss": 2.7843, + "theoretical_loss": 4.207157305988615, + "tokens_seen": 265355264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 157714, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.635972738265991, + "objective/train/theoretical_loss": 4.207032970655965, + "objective/train/tokens_used": 285880800, + "theoretical_loss": 4.207032970655965, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046443329989969914, + "loss": 2.9511, + "theoretical_loss": 4.207032970655965, + "tokens_seen": 265420800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046442326980942827, + "loss": 3.0488, + "theoretical_loss": 4.206908674613213, + "tokens_seen": 265486336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644132397191575, + "loss": 2.7703, + "theoretical_loss": 4.2067844178382465, + "tokens_seen": 265551872 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004644032096288867, + "loss": 2.9723, + "theoretical_loss": 4.206660200308975, + "tokens_seen": 265617408 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046439317953861586, + "loss": 2.896, + "theoretical_loss": 4.206536022003325, + "tokens_seen": 265682944 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046438314944834505, + "loss": 3.1176, + "theoretical_loss": 4.206411882899239, + "tokens_seen": 265748480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643731193580742, + "loss": 3.054, + "theoretical_loss": 4.20628778297468, + "tokens_seen": 265814016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004643630892678034, + "loss": 3.0583, + "theoretical_loss": 4.206163722207627, + "tokens_seen": 265879552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046435305917753264, + "loss": 2.9522, + "theoretical_loss": 4.206039700576078, + "tokens_seen": 265945088 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046434302908726177, + "loss": 3.1155, + "theoretical_loss": 4.205915718058046, + "tokens_seen": 266010624 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464332998996991, + "loss": 3.1129, + "theoretical_loss": 4.205791774631566, + "tokens_seen": 266076160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046432296890672013, + "loss": 3.1271, + "theoretical_loss": 4.205667870274686, + "tokens_seen": 266141696 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046431293881644937, + "loss": 3.0881, + "theoretical_loss": 4.2055440049654775, + "tokens_seen": 266207232 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046430290872617855, + "loss": 3.1299, + "theoretical_loss": 4.205420178682024, + "tokens_seen": 266272768 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046429287863590773, + "loss": 2.8421, + "theoretical_loss": 4.205296391402431, + "tokens_seen": 266338304 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642828485456369, + "loss": 2.9059, + "theoretical_loss": 4.205172643104818, + "tokens_seen": 266403840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046427281845536615, + "loss": 2.8721, + "theoretical_loss": 4.205048933767326, + "tokens_seen": 266469376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642627883650953, + "loss": 3.1183, + "theoretical_loss": 4.20492526336811, + "tokens_seen": 266534912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642527582748245, + "loss": 2.9405, + "theoretical_loss": 4.204801631885346, + "tokens_seen": 266600448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046424272818455364, + "loss": 2.9166, + "theoretical_loss": 4.204678039297223, + "tokens_seen": 266665984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046423269809428287, + "loss": 3.1439, + "theoretical_loss": 4.204554485581953, + "tokens_seen": 266731520 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046422266800401205, + "loss": 3.0142, + "theoretical_loss": 4.2044309707177625, + "tokens_seen": 266797056 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046421263791374123, + "loss": 3.166, + "theoretical_loss": 4.204307494682895, + "tokens_seen": 266862592 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004642026078234704, + "loss": 2.91, + "theoretical_loss": 4.204184057455613, + "tokens_seen": 266928128 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641925777331996, + "loss": 2.7895, + "theoretical_loss": 4.204060659014197, + "tokens_seen": 266993664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 158889, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5656661987304688, + "objective/train/theoretical_loss": 4.2039372993369435, + "objective/train/tokens_used": 287519200, + "theoretical_loss": 4.2039372993369435, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641825476429288, + "loss": 2.9939, + "theoretical_loss": 4.2039372993369435, + "tokens_seen": 267059200 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464172517552658, + "loss": 2.7803, + "theoretical_loss": 4.203813978402167, + "tokens_seen": 267124736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046416248746238714, + "loss": 3.129, + "theoretical_loss": 4.203690696188198, + "tokens_seen": 267190272 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641524573721164, + "loss": 3.0649, + "theoretical_loss": 4.203567452673388, + "tokens_seen": 267255808 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641424272818455, + "loss": 3.0718, + "theoretical_loss": 4.203444247836103, + "tokens_seen": 267321344 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046413239719157474, + "loss": 3.2325, + "theoretical_loss": 4.203321081654726, + "tokens_seen": 267386880 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641223671013039, + "loss": 3.0568, + "theoretical_loss": 4.203197954107661, + "tokens_seen": 267452416 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641123370110331, + "loss": 2.8217, + "theoretical_loss": 4.203074865173325, + "tokens_seen": 267517952 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004641023069207623, + "loss": 2.9454, + "theoretical_loss": 4.202951814830155, + "tokens_seen": 267583488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640922768304915, + "loss": 2.9256, + "theoretical_loss": 4.202828803056605, + "tokens_seen": 267649024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046408224674022064, + "loss": 2.9524, + "theoretical_loss": 4.202705829831144, + "tokens_seen": 267714560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640722166499499, + "loss": 2.9487, + "theoretical_loss": 4.202582895132262, + "tokens_seen": 267780096 + }, + { + "epoch": 0.08, + "learning_rate": 0.000464062186559679, + "loss": 2.6154, + "theoretical_loss": 4.202459998938464, + "tokens_seen": 267845632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046405215646940824, + "loss": 3.114, + "theoretical_loss": 4.202337141228272, + "tokens_seen": 267911168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640421263791374, + "loss": 3.0631, + "theoretical_loss": 4.202214321980227, + "tokens_seen": 267976704 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640320962888666, + "loss": 2.9271, + "theoretical_loss": 4.202091541172885, + "tokens_seen": 268042240 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004640220661985958, + "loss": 2.9358, + "theoretical_loss": 4.201968798784821, + "tokens_seen": 268107776 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046401203610832496, + "loss": 2.9304, + "theoretical_loss": 4.201846094794627, + "tokens_seen": 268173312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046400200601805414, + "loss": 3.2132, + "theoretical_loss": 4.201723429180911, + "tokens_seen": 268238848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639919759277834, + "loss": 2.8554, + "theoretical_loss": 4.2016008019222975, + "tokens_seen": 268304384 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639819458375125, + "loss": 3.0379, + "theoretical_loss": 4.201478212997431, + "tokens_seen": 268369920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046397191574724174, + "loss": 2.8709, + "theoretical_loss": 4.201355662384971, + "tokens_seen": 268435456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639618856569709, + "loss": 3.0275, + "theoretical_loss": 4.2012331500635955, + "tokens_seen": 268500992 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639518555667001, + "loss": 3.1757, + "theoretical_loss": 4.201110676011998, + "tokens_seen": 268566528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046394182547642934, + "loss": 3.3162, + "theoretical_loss": 4.200988240208889, + "tokens_seen": 268632064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 159274, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.012082099914551, + "objective/train/theoretical_loss": 4.2008658426329974, + "objective/train/tokens_used": 289157600, + "theoretical_loss": 4.2008658426329974, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046393179538615847, + "loss": 2.954, + "theoretical_loss": 4.2008658426329974, + "tokens_seen": 268697600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639217652958877, + "loss": 2.9394, + "theoretical_loss": 4.200743483263068, + "tokens_seen": 268763136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004639117352056169, + "loss": 2.9931, + "theoretical_loss": 4.200621162077864, + "tokens_seen": 268828672 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046390170511534606, + "loss": 3.1958, + "theoretical_loss": 4.200498879056164, + "tokens_seen": 268894208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046389167502507525, + "loss": 3.0091, + "theoretical_loss": 4.200376634176765, + "tokens_seen": 268959744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638816449348044, + "loss": 2.9129, + "theoretical_loss": 4.200254427418478, + "tokens_seen": 269025280 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638716148445336, + "loss": 3.0744, + "theoretical_loss": 4.200132258760135, + "tokens_seen": 269090816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046386158475426284, + "loss": 3.0598, + "theoretical_loss": 4.200010128180582, + "tokens_seen": 269156352 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046385155466399197, + "loss": 3.3544, + "theoretical_loss": 4.1998880356586845, + "tokens_seen": 269221888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004638415245737212, + "loss": 3.2152, + "theoretical_loss": 4.1997659811733214, + "tokens_seen": 269287424 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046383149448345033, + "loss": 3.3844, + "theoretical_loss": 4.199643964703391, + "tokens_seen": 269352960 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046382146439317957, + "loss": 3.2429, + "theoretical_loss": 4.199521986227808, + "tokens_seen": 269418496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046381143430290875, + "loss": 3.002, + "theoretical_loss": 4.199400045725502, + "tokens_seen": 269484032 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046380140421263793, + "loss": 2.9083, + "theoretical_loss": 4.199278143175423, + "tokens_seen": 269549568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637913741223671, + "loss": 3.2151, + "theoretical_loss": 4.1991562785565355, + "tokens_seen": 269615104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046378134403209635, + "loss": 2.7854, + "theoretical_loss": 4.199034451847821, + "tokens_seen": 269680640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637713139418255, + "loss": 3.1079, + "theoretical_loss": 4.198912663028278, + "tokens_seen": 269746176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637612838515547, + "loss": 3.2046, + "theoretical_loss": 4.19879091207692, + "tokens_seen": 269811712 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046375125376128384, + "loss": 2.8742, + "theoretical_loss": 4.19866919897278, + "tokens_seen": 269877248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046374122367101307, + "loss": 2.8417, + "theoretical_loss": 4.1985475236949075, + "tokens_seen": 269942784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046373119358074225, + "loss": 3.0719, + "theoretical_loss": 4.198425886222367, + "tokens_seen": 270008320 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046372116349047143, + "loss": 3.1394, + "theoretical_loss": 4.198304286534239, + "tokens_seen": 270073856 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637111334002006, + "loss": 2.8706, + "theoretical_loss": 4.198182724609624, + "tokens_seen": 270139392 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004637011033099298, + "loss": 2.9891, + "theoretical_loss": 4.198061200427635, + "tokens_seen": 270204928 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463691073219659, + "loss": 2.9082, + "theoretical_loss": 4.197939713967407, + "tokens_seen": 270270464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 160520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.315969467163086, + "objective/train/theoretical_loss": 4.197818265208085, + "objective/train/tokens_used": 290796000, + "theoretical_loss": 4.197818265208085, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636810431293882, + "loss": 2.939, + "theoretical_loss": 4.197818265208085, + "tokens_seen": 270336000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046367101303911734, + "loss": 2.9865, + "theoretical_loss": 4.197696854128836, + "tokens_seen": 270401536 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636609829488466, + "loss": 2.9617, + "theoretical_loss": 4.1975754807088395, + "tokens_seen": 270467072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636509528585757, + "loss": 3.1313, + "theoretical_loss": 4.1974541449272955, + "tokens_seen": 270532608 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046364092276830494, + "loss": 2.8589, + "theoretical_loss": 4.197332846763418, + "tokens_seen": 270598144 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636308926780341, + "loss": 3.168, + "theoretical_loss": 4.197211586196437, + "tokens_seen": 270663680 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636208625877633, + "loss": 3.1609, + "theoretical_loss": 4.197090363205602, + "tokens_seen": 270729216 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636108324974925, + "loss": 2.8238, + "theoretical_loss": 4.196969177770175, + "tokens_seen": 270794752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004636008024072217, + "loss": 3.0672, + "theoretical_loss": 4.196848029869437, + "tokens_seen": 270860288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046359077231695084, + "loss": 3.1223, + "theoretical_loss": 4.1967269194826855, + "tokens_seen": 270925824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635807422266801, + "loss": 3.1919, + "theoretical_loss": 4.196605846589234, + "tokens_seen": 270991360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635707121364092, + "loss": 3.1143, + "theoretical_loss": 4.196484811168412, + "tokens_seen": 271056896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046356068204613844, + "loss": 2.7825, + "theoretical_loss": 4.196363813199565, + "tokens_seen": 271122432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635506519558676, + "loss": 3.1101, + "theoretical_loss": 4.196242852662056, + "tokens_seen": 271187968 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635406218655968, + "loss": 3.1081, + "theoretical_loss": 4.1961219295352645, + "tokens_seen": 271253504 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463530591775326, + "loss": 3.1529, + "theoretical_loss": 4.196001043798585, + "tokens_seen": 271319040 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046352056168505516, + "loss": 3.1475, + "theoretical_loss": 4.195880195431428, + "tokens_seen": 271384576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046351053159478434, + "loss": 2.9622, + "theoretical_loss": 4.195759384413224, + "tokens_seen": 271450112 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004635005015045136, + "loss": 2.9907, + "theoretical_loss": 4.195638610723415, + "tokens_seen": 271515648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634904714142427, + "loss": 2.7124, + "theoretical_loss": 4.195517874341462, + "tokens_seen": 271581184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046348044132397194, + "loss": 2.9731, + "theoretical_loss": 4.195397175246841, + "tokens_seen": 271646720 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046347041123370107, + "loss": 3.1014, + "theoretical_loss": 4.195276513419046, + "tokens_seen": 271712256 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634603811434303, + "loss": 2.8613, + "theoretical_loss": 4.195155888837585, + "tokens_seen": 271777792 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634503510531595, + "loss": 3.0449, + "theoretical_loss": 4.195035301481984, + "tokens_seen": 271843328 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046344032096288867, + "loss": 2.7986, + "theoretical_loss": 4.194914751331785, + "tokens_seen": 271908864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 160942, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2946550846099854, + "objective/train/theoretical_loss": 4.194794238366544, + "objective/train/tokens_used": 292434400, + "theoretical_loss": 4.194794238366544, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046343029087261785, + "loss": 3.0722, + "theoretical_loss": 4.194794238366544, + "tokens_seen": 271974400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634202607823471, + "loss": 3.0775, + "theoretical_loss": 4.194673762565837, + "tokens_seen": 272039936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004634102306920762, + "loss": 2.9018, + "theoretical_loss": 4.194553323909252, + "tokens_seen": 272105472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046340020060180545, + "loss": 3.1252, + "theoretical_loss": 4.1944329223763965, + "tokens_seen": 272171008 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046339017051153457, + "loss": 2.9432, + "theoretical_loss": 4.194312557946892, + "tokens_seen": 272236544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633801404212638, + "loss": 3.1044, + "theoretical_loss": 4.194192230600376, + "tokens_seen": 272302080 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463370110330993, + "loss": 3.0371, + "theoretical_loss": 4.194071940316505, + "tokens_seen": 272367616 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046336008024072217, + "loss": 3.1066, + "theoretical_loss": 4.193951687074948, + "tokens_seen": 272433152 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046335005015045135, + "loss": 2.9569, + "theoretical_loss": 4.193831470855391, + "tokens_seen": 272498688 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046334002006018053, + "loss": 3.1706, + "theoretical_loss": 4.193711291637538, + "tokens_seen": 272564224 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633299899699097, + "loss": 3.0875, + "theoretical_loss": 4.1935911494011044, + "tokens_seen": 272629760 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046331995987963895, + "loss": 3.0274, + "theoretical_loss": 4.193471044125828, + "tokens_seen": 272695296 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004633099297893681, + "loss": 3.1833, + "theoretical_loss": 4.193350975791458, + "tokens_seen": 272760832 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632998996990973, + "loss": 3.0265, + "theoretical_loss": 4.193230944377762, + "tokens_seen": 272826368 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046328986960882644, + "loss": 2.999, + "theoretical_loss": 4.1931109498645185, + "tokens_seen": 272891904 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632798395185557, + "loss": 2.8691, + "theoretical_loss": 4.19299099223153, + "tokens_seen": 272957440 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046326980942828485, + "loss": 3.1468, + "theoretical_loss": 4.192871071458608, + "tokens_seen": 273022976 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046325977933801404, + "loss": 3.101, + "theoretical_loss": 4.192751187525585, + "tokens_seen": 273088512 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632497492477432, + "loss": 2.8904, + "theoretical_loss": 4.192631340412304, + "tokens_seen": 273154048 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046323971915747245, + "loss": 2.8277, + "theoretical_loss": 4.192511530098629, + "tokens_seen": 273219584 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632296890672016, + "loss": 3.0174, + "theoretical_loss": 4.192391756564437, + "tokens_seen": 273285120 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004632196589769308, + "loss": 2.9337, + "theoretical_loss": 4.19227201978962, + "tokens_seen": 273350656 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046320962888666, + "loss": 3.2491, + "theoretical_loss": 4.19215231975409, + "tokens_seen": 273416192 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631995987963892, + "loss": 3.1248, + "theoretical_loss": 4.192032656437769, + "tokens_seen": 273481728 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631895687061184, + "loss": 3.1044, + "theoretical_loss": 4.191913029820601, + "tokens_seen": 273547264 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 162274, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1903703212738037, + "objective/train/theoretical_loss": 4.19179343988254, + "objective/train/tokens_used": 294072800, + "theoretical_loss": 4.19179343988254, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046317953861584754, + "loss": 3.1483, + "theoretical_loss": 4.19179343988254, + "tokens_seen": 273612800 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631695085255768, + "loss": 2.8653, + "theoretical_loss": 4.191673886603559, + "tokens_seen": 273678336 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631594784353059, + "loss": 3.0805, + "theoretical_loss": 4.191554369963647, + "tokens_seen": 273743872 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046314944834503514, + "loss": 2.9344, + "theoretical_loss": 4.191434889942807, + "tokens_seen": 273809408 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631394182547643, + "loss": 3.0085, + "theoretical_loss": 4.191315446521058, + "tokens_seen": 273874944 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631293881644935, + "loss": 2.9289, + "theoretical_loss": 4.191196039678436, + "tokens_seen": 273940480 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631193580742227, + "loss": 3.1368, + "theoretical_loss": 4.191076669394992, + "tokens_seen": 274006016 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004631093279839519, + "loss": 3.1613, + "theoretical_loss": 4.190957335650791, + "tokens_seen": 274071552 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046309929789368104, + "loss": 2.8155, + "theoretical_loss": 4.190838038425916, + "tokens_seen": 274137088 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630892678034103, + "loss": 3.1902, + "theoretical_loss": 4.190718777700465, + "tokens_seen": 274202624 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630792377131394, + "loss": 3.0513, + "theoretical_loss": 4.190599553454551, + "tokens_seen": 274268160 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046306920762286864, + "loss": 3.0037, + "theoretical_loss": 4.190480365668304, + "tokens_seen": 274333696 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630591775325978, + "loss": 2.8063, + "theoretical_loss": 4.190361214321866, + "tokens_seen": 274399232 + }, + { + "epoch": 0.08, + "learning_rate": 0.000463049147442327, + "loss": 2.9742, + "theoretical_loss": 4.190242099395399, + "tokens_seen": 274464768 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630391173520562, + "loss": 2.9558, + "theoretical_loss": 4.190123020869077, + "tokens_seen": 274530304 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046302908726178536, + "loss": 2.9187, + "theoretical_loss": 4.190003978723093, + "tokens_seen": 274595840 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046301905717151455, + "loss": 3.0974, + "theoretical_loss": 4.189884972937652, + "tokens_seen": 274661376 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004630090270812438, + "loss": 3.1408, + "theoretical_loss": 4.189766003492977, + "tokens_seen": 274726912 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629989969909729, + "loss": 3.0551, + "theoretical_loss": 4.189647070369304, + "tokens_seen": 274792448 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046298896690070214, + "loss": 3.1843, + "theoretical_loss": 4.189528173546888, + "tokens_seen": 274857984 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046297893681043127, + "loss": 3.0686, + "theoretical_loss": 4.189409313005997, + "tokens_seen": 274923520 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629689067201605, + "loss": 3.0667, + "theoretical_loss": 4.189290488726913, + "tokens_seen": 274989056 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629588766298897, + "loss": 3.2895, + "theoretical_loss": 4.189171700689937, + "tokens_seen": 275054592 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046294884653961887, + "loss": 3.0592, + "theoretical_loss": 4.189052948875384, + "tokens_seen": 275120128 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046293881644934805, + "loss": 2.9131, + "theoretical_loss": 4.188934233263582, + "tokens_seen": 275185664 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 163440, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.128859519958496, + "objective/train/theoretical_loss": 4.188815553834879, + "objective/train/tokens_used": 295711200, + "theoretical_loss": 4.188815553834879, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629287863590773, + "loss": 3.0631, + "theoretical_loss": 4.188815553834879, + "tokens_seen": 275251200 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004629187562688064, + "loss": 2.8185, + "theoretical_loss": 4.188696910569634, + "tokens_seen": 275316736 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046290872617853565, + "loss": 3.0597, + "theoretical_loss": 4.188578303448223, + "tokens_seen": 275382272 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046289869608826477, + "loss": 3.0644, + "theoretical_loss": 4.188459732451038, + "tokens_seen": 275447808 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462888665997994, + "loss": 3.2541, + "theoretical_loss": 4.188341197558486, + "tokens_seen": 275513344 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628786359077232, + "loss": 2.772, + "theoretical_loss": 4.188222698750988, + "tokens_seen": 275578880 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046286860581745237, + "loss": 2.945, + "theoretical_loss": 4.188104236008982, + "tokens_seen": 275644416 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046285857572718155, + "loss": 2.9857, + "theoretical_loss": 4.18798580931292, + "tokens_seen": 275709952 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046284854563691073, + "loss": 2.8546, + "theoretical_loss": 4.18786741864327, + "tokens_seen": 275775488 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628385155466399, + "loss": 3.1056, + "theoretical_loss": 4.187749063980516, + "tokens_seen": 275841024 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046282848545636915, + "loss": 2.8899, + "theoretical_loss": 4.187630745305154, + "tokens_seen": 275906560 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628184553660983, + "loss": 2.9852, + "theoretical_loss": 4.187512462597699, + "tokens_seen": 275972096 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004628084252758275, + "loss": 3.1059, + "theoretical_loss": 4.187394215838678, + "tokens_seen": 276037632 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046279839518555664, + "loss": 2.945, + "theoretical_loss": 4.187276005008637, + "tokens_seen": 276103168 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627883650952859, + "loss": 3.1584, + "theoretical_loss": 4.1871578300881325, + "tokens_seen": 276168704 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046277833500501505, + "loss": 3.1305, + "theoretical_loss": 4.18703969105774, + "tokens_seen": 276234240 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046276830491474424, + "loss": 2.8816, + "theoretical_loss": 4.186921587898048, + "tokens_seen": 276299776 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627582748244734, + "loss": 2.988, + "theoretical_loss": 4.1868035205896605, + "tokens_seen": 276365312 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046274824473420265, + "loss": 2.8155, + "theoretical_loss": 4.186685489113197, + "tokens_seen": 276430848 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627382146439318, + "loss": 3.1242, + "theoretical_loss": 4.186567493449292, + "tokens_seen": 276496384 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462728184553661, + "loss": 3.0243, + "theoretical_loss": 4.186449533578595, + "tokens_seen": 276561920 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046271815446339014, + "loss": 2.9596, + "theoretical_loss": 4.186331609481769, + "tokens_seen": 276627456 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004627081243731194, + "loss": 3.3328, + "theoretical_loss": 4.186213721139496, + "tokens_seen": 276692992 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046269809428284856, + "loss": 3.114, + "theoretical_loss": 4.186095868532469, + "tokens_seen": 276758528 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046268806419257774, + "loss": 2.8937, + "theoretical_loss": 4.1859780516413965, + "tokens_seen": 276824064 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 163970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.389463186264038, + "objective/train/theoretical_loss": 4.185860270447004, + "objective/train/tokens_used": 297349600, + "theoretical_loss": 4.185860270447004, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626780341023069, + "loss": 3.0341, + "theoretical_loss": 4.185860270447004, + "tokens_seen": 276889600 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626680040120361, + "loss": 3.1028, + "theoretical_loss": 4.185742524930031, + "tokens_seen": 276955136 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626579739217653, + "loss": 2.9541, + "theoretical_loss": 4.185624815071232, + "tokens_seen": 277020672 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626479438314945, + "loss": 2.9788, + "theoretical_loss": 4.185507140851375, + "tokens_seen": 277086208 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046263791374122364, + "loss": 3.0129, + "theoretical_loss": 4.185389502251246, + "tokens_seen": 277151744 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004626278836509529, + "loss": 2.8961, + "theoretical_loss": 4.185271899251642, + "tokens_seen": 277217280 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462617853560682, + "loss": 2.8754, + "theoretical_loss": 4.185154331833379, + "tokens_seen": 277282816 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046260782347041124, + "loss": 2.9643, + "theoretical_loss": 4.185036799977285, + "tokens_seen": 277348352 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625977933801404, + "loss": 3.0866, + "theoretical_loss": 4.184919303664203, + "tokens_seen": 277413888 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625877632898696, + "loss": 3.1173, + "theoretical_loss": 4.184801842874992, + "tokens_seen": 277479424 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625777331995988, + "loss": 3.0844, + "theoretical_loss": 4.1846844175905265, + "tokens_seen": 277544960 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462567703109328, + "loss": 3.0707, + "theoretical_loss": 4.184567027791693, + "tokens_seen": 277610496 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046255767301905715, + "loss": 2.9221, + "theoretical_loss": 4.184449673459395, + "tokens_seen": 277676032 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625476429287864, + "loss": 3.0805, + "theoretical_loss": 4.1843323545745505, + "tokens_seen": 277741568 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625376128385155, + "loss": 2.9624, + "theoretical_loss": 4.184215071118093, + "tokens_seen": 277807104 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046252758274824475, + "loss": 2.9347, + "theoretical_loss": 4.184097823070967, + "tokens_seen": 277872640 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625175526579739, + "loss": 3.0535, + "theoretical_loss": 4.183980610414137, + "tokens_seen": 277938176 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004625075225677031, + "loss": 2.9498, + "theoretical_loss": 4.18386343312858, + "tokens_seen": 278003712 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624974924774323, + "loss": 2.9249, + "theoretical_loss": 4.183746291195285, + "tokens_seen": 278069248 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046248746238716147, + "loss": 3.0889, + "theoretical_loss": 4.18362918459526, + "tokens_seen": 278134784 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046247743229689065, + "loss": 2.9025, + "theoretical_loss": 4.183512113309525, + "tokens_seen": 278200320 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624674022066199, + "loss": 3.1737, + "theoretical_loss": 4.1833950773191155, + "tokens_seen": 278265856 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046245737211634907, + "loss": 2.8199, + "theoretical_loss": 4.183278076605083, + "tokens_seen": 278331392 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046244734202607825, + "loss": 3.1762, + "theoretical_loss": 4.18316111114849, + "tokens_seen": 278396928 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624373119358075, + "loss": 3.1915, + "theoretical_loss": 4.183044180930418, + "tokens_seen": 278462464 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 165182, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.31074857711792, + "objective/train/theoretical_loss": 4.182927285931959, + "objective/train/tokens_used": 298988000, + "theoretical_loss": 4.182927285931959, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004624272818455366, + "loss": 3.0716, + "theoretical_loss": 4.182927285931959, + "tokens_seen": 278528000 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046241725175526585, + "loss": 2.8782, + "theoretical_loss": 4.182810426134223, + "tokens_seen": 278593536 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462407221664995, + "loss": 3.1372, + "theoretical_loss": 4.1826936015183325, + "tokens_seen": 278659072 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623971915747242, + "loss": 3.0399, + "theoretical_loss": 4.182576812065426, + "tokens_seen": 278724608 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623871614844534, + "loss": 3.1069, + "theoretical_loss": 4.1824600577566535, + "tokens_seen": 278790144 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046237713139418257, + "loss": 3.003, + "theoretical_loss": 4.182343338573185, + "tokens_seen": 278855680 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046236710130391175, + "loss": 2.981, + "theoretical_loss": 4.1822266544961995, + "tokens_seen": 278921216 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046235707121364093, + "loss": 2.9963, + "theoretical_loss": 4.1821100055068925, + "tokens_seen": 278986752 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623470411233701, + "loss": 2.8282, + "theoretical_loss": 4.181993391586476, + "tokens_seen": 279052288 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046233701103309935, + "loss": 2.9174, + "theoretical_loss": 4.181876812716174, + "tokens_seen": 279117824 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623269809428285, + "loss": 3.0528, + "theoretical_loss": 4.181760268877226, + "tokens_seen": 279183360 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004623169508525577, + "loss": 3.0571, + "theoretical_loss": 4.181643760050885, + "tokens_seen": 279248896 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046230692076228684, + "loss": 3.0749, + "theoretical_loss": 4.181527286218419, + "tokens_seen": 279314432 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622968906720161, + "loss": 3.0104, + "theoretical_loss": 4.181410847361111, + "tokens_seen": 279379968 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046228686058174525, + "loss": 3.1389, + "theoretical_loss": 4.181294443460257, + "tokens_seen": 279445504 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046227683049147444, + "loss": 2.8566, + "theoretical_loss": 4.18117807449717, + "tokens_seen": 279511040 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622668004012036, + "loss": 3.1739, + "theoretical_loss": 4.181061740453174, + "tokens_seen": 279576576 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046225677031093285, + "loss": 3.1752, + "theoretical_loss": 4.18094544130961, + "tokens_seen": 279642112 + }, + { + "epoch": 0.08, + "learning_rate": 0.000462246740220662, + "loss": 2.9331, + "theoretical_loss": 4.180829177047832, + "tokens_seen": 279707648 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622367101303912, + "loss": 2.8764, + "theoretical_loss": 4.180712947649208, + "tokens_seen": 279773184 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046222668004012034, + "loss": 3.1994, + "theoretical_loss": 4.180596753095122, + "tokens_seen": 279838720 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004622166499498496, + "loss": 3.2665, + "theoretical_loss": 4.18048059336697, + "tokens_seen": 279904256 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046220661985957876, + "loss": 2.9311, + "theoretical_loss": 4.180364468446165, + "tokens_seen": 279969792 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046219658976930794, + "loss": 3.0726, + "theoretical_loss": 4.180248378314133, + "tokens_seen": 280035328 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621865596790371, + "loss": 3.0656, + "theoretical_loss": 4.180132322952312, + "tokens_seen": 280100864 + }, + { + "epoch": 0.08, + "objective/train/docs_used": 165816, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.640207529067993, + "objective/train/theoretical_loss": 4.180016302342159, + "objective/train/tokens_used": 300626400, + "theoretical_loss": 4.180016302342159, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621765295887663, + "loss": 2.8359, + "theoretical_loss": 4.180016302342159, + "tokens_seen": 280166400 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621664994984955, + "loss": 3.0966, + "theoretical_loss": 4.179900316465139, + "tokens_seen": 280231936 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621564694082247, + "loss": 3.0808, + "theoretical_loss": 4.179784365302739, + "tokens_seen": 280297472 + }, + { + "epoch": 0.08, + "learning_rate": 0.00046214643931795384, + "loss": 2.9101, + "theoretical_loss": 4.179668448836454, + "tokens_seen": 280363008 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621364092276831, + "loss": 3.1295, + "theoretical_loss": 4.179552567047794, + "tokens_seen": 280428544 + }, + { + "epoch": 0.08, + "learning_rate": 0.0004621263791374122, + "loss": 2.9127, + "theoretical_loss": 4.179436719918286, + "tokens_seen": 280494080 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046211634904714144, + "loss": 3.2209, + "theoretical_loss": 4.179320907429468, + "tokens_seen": 280559616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004621063189568706, + "loss": 3.0106, + "theoretical_loss": 4.179205129562895, + "tokens_seen": 280625152 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620962888665998, + "loss": 3.0269, + "theoretical_loss": 4.179089386300134, + "tokens_seen": 280690688 + }, + { + "epoch": 0.09, + "learning_rate": 0.000462086258776329, + "loss": 3.1179, + "theoretical_loss": 4.178973677622768, + "tokens_seen": 280756224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620762286860582, + "loss": 3.278, + "theoretical_loss": 4.17885800351239, + "tokens_seen": 280821760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046206619859578735, + "loss": 2.9086, + "theoretical_loss": 4.178742363950614, + "tokens_seen": 280887296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620561685055166, + "loss": 3.1124, + "theoretical_loss": 4.178626758919062, + "tokens_seen": 280952832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620461384152457, + "loss": 3.0546, + "theoretical_loss": 4.178511188399372, + "tokens_seen": 281018368 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046203610832497495, + "loss": 3.1334, + "theoretical_loss": 4.1783956523731955, + "tokens_seen": 281083904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620260782347041, + "loss": 2.9334, + "theoretical_loss": 4.178280150822202, + "tokens_seen": 281149440 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620160481444333, + "loss": 2.8178, + "theoretical_loss": 4.178164683728068, + "tokens_seen": 281214976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004620060180541625, + "loss": 2.8805, + "theoretical_loss": 4.17804925107249, + "tokens_seen": 281280512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046199598796389167, + "loss": 2.919, + "theoretical_loss": 4.1779338528371746, + "tokens_seen": 281346048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046198595787362085, + "loss": 3.1013, + "theoretical_loss": 4.177818489003846, + "tokens_seen": 281411584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619759277833501, + "loss": 3.1344, + "theoretical_loss": 4.177703159554238, + "tokens_seen": 281477120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619658976930792, + "loss": 3.0054, + "theoretical_loss": 4.177587864470103, + "tokens_seen": 281542656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046195586760280845, + "loss": 2.8447, + "theoretical_loss": 4.1774726037332055, + "tokens_seen": 281608192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046194583751253763, + "loss": 3.0365, + "theoretical_loss": 4.17735737732532, + "tokens_seen": 281673728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619358074222668, + "loss": 3.0124, + "theoretical_loss": 4.177242185228242, + "tokens_seen": 281739264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 166963, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.019186019897461, + "objective/train/theoretical_loss": 4.177127027423776, + "objective/train/tokens_used": 302264800, + "theoretical_loss": 4.177127027423776, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461925777331996, + "loss": 2.9856, + "theoretical_loss": 4.177127027423776, + "tokens_seen": 281804800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004619157472417252, + "loss": 3.0215, + "theoretical_loss": 4.17701190389374, + "tokens_seen": 281870336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046190571715145435, + "loss": 2.9971, + "theoretical_loss": 4.176896814619969, + "tokens_seen": 281935872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618956870611836, + "loss": 3.0423, + "theoretical_loss": 4.176781759584312, + "tokens_seen": 282001408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618856569709127, + "loss": 3.0798, + "theoretical_loss": 4.176666738768627, + "tokens_seen": 282066944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046187562688064195, + "loss": 2.8872, + "theoretical_loss": 4.176551752154791, + "tokens_seen": 282132480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618655967903711, + "loss": 3.1228, + "theoretical_loss": 4.1764367997246925, + "tokens_seen": 282198016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618555667001003, + "loss": 3.0256, + "theoretical_loss": 4.1763218814602325, + "tokens_seen": 282263552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618455366098295, + "loss": 3.1382, + "theoretical_loss": 4.17620699734333, + "tokens_seen": 282329088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618355065195587, + "loss": 2.807, + "theoretical_loss": 4.176092147355915, + "tokens_seen": 282394624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046182547642928786, + "loss": 2.9633, + "theoretical_loss": 4.175977331479929, + "tokens_seen": 282460160 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046181544633901704, + "loss": 3.0836, + "theoretical_loss": 4.175862549697332, + "tokens_seen": 282525696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004618054162487462, + "loss": 3.0762, + "theoretical_loss": 4.175747801990093, + "tokens_seen": 282591232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046179538615847545, + "loss": 3.1594, + "theoretical_loss": 4.1756330883402, + "tokens_seen": 282656768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617853560682046, + "loss": 3.136, + "theoretical_loss": 4.1755184087296495, + "tokens_seen": 282722304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617753259779338, + "loss": 3.0347, + "theoretical_loss": 4.175403763140456, + "tokens_seen": 282787840 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461765295887663, + "loss": 2.9652, + "theoretical_loss": 4.175289151554643, + "tokens_seen": 282853376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617552657973922, + "loss": 3.0566, + "theoretical_loss": 4.175174573954252, + "tokens_seen": 282918912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046174523570712136, + "loss": 2.8965, + "theoretical_loss": 4.175060030321337, + "tokens_seen": 282984448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046173520561685054, + "loss": 3.0043, + "theoretical_loss": 4.174945520637964, + "tokens_seen": 283049984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004617251755265797, + "loss": 3.124, + "theoretical_loss": 4.174831044886213, + "tokens_seen": 283115520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046171514543630896, + "loss": 3.0059, + "theoretical_loss": 4.17471660304818, + "tokens_seen": 283181056 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046170511534603814, + "loss": 3.1171, + "theoretical_loss": 4.174602195105972, + "tokens_seen": 283246592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616950852557673, + "loss": 2.735, + "theoretical_loss": 4.174487821041711, + "tokens_seen": 283312128 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616850551654965, + "loss": 2.9887, + "theoretical_loss": 4.174373480837532, + "tokens_seen": 283377664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 167624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3214924335479736, + "objective/train/theoretical_loss": 4.174259174475583, + "objective/train/tokens_used": 303903200, + "theoretical_loss": 4.174259174475583, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616750250752257, + "loss": 3.0628, + "theoretical_loss": 4.174259174475583, + "tokens_seen": 283443200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616649949849549, + "loss": 3.1753, + "theoretical_loss": 4.174144901938025, + "tokens_seen": 283508736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046165496489468404, + "loss": 2.9032, + "theoretical_loss": 4.174030663207036, + "tokens_seen": 283574272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616449348044133, + "loss": 3.0913, + "theoretical_loss": 4.173916458264804, + "tokens_seen": 283639808 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616349047141424, + "loss": 3.0559, + "theoretical_loss": 4.173802287093531, + "tokens_seen": 283705344 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046162487462387164, + "loss": 2.8405, + "theoretical_loss": 4.173688149675435, + "tokens_seen": 283770880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004616148445336008, + "loss": 3.1749, + "theoretical_loss": 4.173574045992744, + "tokens_seen": 283836416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046160481444333, + "loss": 2.9452, + "theoretical_loss": 4.173459976027702, + "tokens_seen": 283901952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615947843530592, + "loss": 3.1586, + "theoretical_loss": 4.173345939762564, + "tokens_seen": 283967488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615847542627884, + "loss": 2.9694, + "theoretical_loss": 4.173231937179601, + "tokens_seen": 284033024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046157472417251755, + "loss": 3.2162, + "theoretical_loss": 4.173117968261097, + "tokens_seen": 284098560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615646940822468, + "loss": 3.1405, + "theoretical_loss": 4.173004032989347, + "tokens_seen": 284164096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615546639919759, + "loss": 3.0749, + "theoretical_loss": 4.172890131346662, + "tokens_seen": 284229632 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046154463390170515, + "loss": 3.1627, + "theoretical_loss": 4.172776263315367, + "tokens_seen": 284295168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615346038114343, + "loss": 3.1345, + "theoretical_loss": 4.172662428877796, + "tokens_seen": 284360704 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615245737211635, + "loss": 2.9964, + "theoretical_loss": 4.172548628016302, + "tokens_seen": 284426240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004615145436308927, + "loss": 3.0997, + "theoretical_loss": 4.172434860713246, + "tokens_seen": 284491776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046150451354062187, + "loss": 3.1053, + "theoretical_loss": 4.172321126951008, + "tokens_seen": 284557312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046149448345035105, + "loss": 3.108, + "theoretical_loss": 4.172207426711974, + "tokens_seen": 284622848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614844533600803, + "loss": 2.8824, + "theoretical_loss": 4.1720937599785515, + "tokens_seen": 284688384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614744232698094, + "loss": 3.2381, + "theoretical_loss": 4.171980126733155, + "tokens_seen": 284753920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046146439317953865, + "loss": 3.1941, + "theoretical_loss": 4.171866526958215, + "tokens_seen": 284819456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046145436308926783, + "loss": 2.7743, + "theoretical_loss": 4.171752960636176, + "tokens_seen": 284884992 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461444332998997, + "loss": 2.9867, + "theoretical_loss": 4.171639427749492, + "tokens_seen": 284950528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614343029087262, + "loss": 3.0296, + "theoretical_loss": 4.171525928280635, + "tokens_seen": 285016064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 168853, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6973540782928467, + "objective/train/theoretical_loss": 4.171412462212087, + "objective/train/tokens_used": 305541600, + "theoretical_loss": 4.171412462212087, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614242728184554, + "loss": 2.8165, + "theoretical_loss": 4.171412462212087, + "tokens_seen": 285081600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046141424272818455, + "loss": 3.1382, + "theoretical_loss": 4.171299029526344, + "tokens_seen": 285147136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004614042126379138, + "loss": 3.1788, + "theoretical_loss": 4.171185630205917, + "tokens_seen": 285212672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613941825476429, + "loss": 2.9897, + "theoretical_loss": 4.171072264233327, + "tokens_seen": 285278208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046138415245737215, + "loss": 3.0584, + "theoretical_loss": 4.17095893159111, + "tokens_seen": 285343744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613741223671013, + "loss": 3.0978, + "theoretical_loss": 4.170845632261815, + "tokens_seen": 285409280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613640922768305, + "loss": 2.9229, + "theoretical_loss": 4.1707323662280045, + "tokens_seen": 285474816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613540621865597, + "loss": 3.1211, + "theoretical_loss": 4.170619133472253, + "tokens_seen": 285540352 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613440320962889, + "loss": 2.9585, + "theoretical_loss": 4.170505933977149, + "tokens_seen": 285605888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046133400200601806, + "loss": 3.1654, + "theoretical_loss": 4.1703927677252945, + "tokens_seen": 285671424 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046132397191574724, + "loss": 3.116, + "theoretical_loss": 4.170279634699304, + "tokens_seen": 285736960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004613139418254764, + "loss": 3.2018, + "theoretical_loss": 4.170166534881803, + "tokens_seen": 285802496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046130391173520566, + "loss": 3.0549, + "theoretical_loss": 4.170053468255435, + "tokens_seen": 285868032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612938816449348, + "loss": 2.795, + "theoretical_loss": 4.169940434802851, + "tokens_seen": 285933568 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461283851554664, + "loss": 3.0738, + "theoretical_loss": 4.16982743450672, + "tokens_seen": 285999104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612738214643932, + "loss": 3.0826, + "theoretical_loss": 4.169714467349721, + "tokens_seen": 286064640 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612637913741224, + "loss": 2.9928, + "theoretical_loss": 4.1696015333145455, + "tokens_seen": 286130176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046125376128385156, + "loss": 3.0505, + "theoretical_loss": 4.169488632383901, + "tokens_seen": 286195712 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046124373119358074, + "loss": 3.1082, + "theoretical_loss": 4.169375764540505, + "tokens_seen": 286261248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612337011033099, + "loss": 2.9721, + "theoretical_loss": 4.16926292976709, + "tokens_seen": 286326784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046122367101303916, + "loss": 3.1107, + "theoretical_loss": 4.1691501280464, + "tokens_seen": 286392320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612136409227683, + "loss": 3.0903, + "theoretical_loss": 4.1690373593611945, + "tokens_seen": 286457856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004612036108324975, + "loss": 2.9085, + "theoretical_loss": 4.168924623694241, + "tokens_seen": 286523392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046119358074222665, + "loss": 2.9324, + "theoretical_loss": 4.168811921028325, + "tokens_seen": 286588928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611835506519559, + "loss": 2.9307, + "theoretical_loss": 4.168699251346243, + "tokens_seen": 286654464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 169449, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8505656719207764, + "objective/train/theoretical_loss": 4.168586614630803, + "objective/train/tokens_used": 307180000, + "theoretical_loss": 4.168586614630803, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046117352056168506, + "loss": 3.006, + "theoretical_loss": 4.168586614630803, + "tokens_seen": 286720000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046116349047141425, + "loss": 2.8379, + "theoretical_loss": 4.168474010864829, + "tokens_seen": 286785536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611534603811434, + "loss": 2.8133, + "theoretical_loss": 4.168361440031154, + "tokens_seen": 286851072 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611434302908726, + "loss": 2.8832, + "theoretical_loss": 4.1682489021126266, + "tokens_seen": 286916608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611334002006018, + "loss": 3.0822, + "theoretical_loss": 4.168136397092109, + "tokens_seen": 286982144 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461123370110331, + "loss": 3.1401, + "theoretical_loss": 4.168023924952473, + "tokens_seen": 287047680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046111334002006015, + "loss": 3.0236, + "theoretical_loss": 4.167911485676606, + "tokens_seen": 287113216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004611033099297894, + "loss": 2.9008, + "theoretical_loss": 4.1677990792474064, + "tokens_seen": 287178752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046109327983951857, + "loss": 3.0368, + "theoretical_loss": 4.167686705647787, + "tokens_seen": 287244288 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046108324974924775, + "loss": 2.8824, + "theoretical_loss": 4.167574364860673, + "tokens_seen": 287309824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046107321965897693, + "loss": 2.8937, + "theoretical_loss": 4.167462056869001, + "tokens_seen": 287375360 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610631895687061, + "loss": 3.037, + "theoretical_loss": 4.167349781655721, + "tokens_seen": 287440896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610531594784353, + "loss": 3.0067, + "theoretical_loss": 4.167237539203798, + "tokens_seen": 287506432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610431293881645, + "loss": 2.8215, + "theoretical_loss": 4.167125329496207, + "tokens_seen": 287571968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046103309929789365, + "loss": 3.092, + "theoretical_loss": 4.167013152515935, + "tokens_seen": 287637504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004610230692076229, + "loss": 2.834, + "theoretical_loss": 4.1669010082459845, + "tokens_seen": 287703040 + }, + { + "epoch": 0.09, + "learning_rate": 0.000461013039117352, + "loss": 3.0378, + "theoretical_loss": 4.16678889666937, + "tokens_seen": 287768576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046100300902708125, + "loss": 2.9538, + "theoretical_loss": 4.166676817769118, + "tokens_seen": 287834112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046099297893681043, + "loss": 2.936, + "theoretical_loss": 4.166564771528268, + "tokens_seen": 287899648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609829488465396, + "loss": 3.0104, + "theoretical_loss": 4.16645275792987, + "tokens_seen": 287965184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609729187562688, + "loss": 3.0223, + "theoretical_loss": 4.166340776956991, + "tokens_seen": 288030720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046096288866599803, + "loss": 2.9447, + "theoretical_loss": 4.166228828592708, + "tokens_seen": 288096256 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609528585757272, + "loss": 2.9425, + "theoretical_loss": 4.166116912820109, + "tokens_seen": 288161792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609428284854564, + "loss": 3.0139, + "theoretical_loss": 4.166005029622298, + "tokens_seen": 288227328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609327983951856, + "loss": 2.7581, + "theoretical_loss": 4.16589317898239, + "tokens_seen": 288292864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 170373, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9227163791656494, + "objective/train/theoretical_loss": 4.165781360883513, + "objective/train/tokens_used": 308818400, + "theoretical_loss": 4.165781360883513, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046092276830491475, + "loss": 3.0162, + "theoretical_loss": 4.165781360883513, + "tokens_seen": 288358400 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460912738214644, + "loss": 2.9972, + "theoretical_loss": 4.165669575308807, + "tokens_seen": 288423936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004609027081243731, + "loss": 2.9231, + "theoretical_loss": 4.165557822241425, + "tokens_seen": 288489472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046089267803410235, + "loss": 2.9746, + "theoretical_loss": 4.165446101664531, + "tokens_seen": 288555008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608826479438315, + "loss": 3.0331, + "theoretical_loss": 4.165334413561306, + "tokens_seen": 288620544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608726178535607, + "loss": 3.0425, + "theoretical_loss": 4.165222757914936, + "tokens_seen": 288686080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608625877632899, + "loss": 2.9917, + "theoretical_loss": 4.165111134708629, + "tokens_seen": 288751616 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608525576730191, + "loss": 3.0351, + "theoretical_loss": 4.164999543925597, + "tokens_seen": 288817152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046084252758274826, + "loss": 2.9461, + "theoretical_loss": 4.164887985549068, + "tokens_seen": 288882688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046083249749247744, + "loss": 3.0879, + "theoretical_loss": 4.164776459562285, + "tokens_seen": 288948224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004608224674022066, + "loss": 2.9828, + "theoretical_loss": 4.164664965948499, + "tokens_seen": 289013760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046081243731193586, + "loss": 3.1005, + "theoretical_loss": 4.164553504690976, + "tokens_seen": 289079296 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460802407221665, + "loss": 3.232, + "theoretical_loss": 4.164442075772993, + "tokens_seen": 289144832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607923771313942, + "loss": 3.0206, + "theoretical_loss": 4.164330679177841, + "tokens_seen": 289210368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607823470411234, + "loss": 2.9927, + "theoretical_loss": 4.164219314888823, + "tokens_seen": 289275904 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607723169508526, + "loss": 2.8879, + "theoretical_loss": 4.164107982889252, + "tokens_seen": 289341440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046076228686058176, + "loss": 2.8926, + "theoretical_loss": 4.163996683162458, + "tokens_seen": 289406976 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046075225677031094, + "loss": 2.955, + "theoretical_loss": 4.163885415691779, + "tokens_seen": 289472512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607422266800401, + "loss": 2.9791, + "theoretical_loss": 4.163774180460569, + "tokens_seen": 289538048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046073219658976936, + "loss": 2.9538, + "theoretical_loss": 4.163662977452192, + "tokens_seen": 289603584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607221664994985, + "loss": 2.9461, + "theoretical_loss": 4.163551806650023, + "tokens_seen": 289669120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004607121364092277, + "loss": 2.8711, + "theoretical_loss": 4.163440668037453, + "tokens_seen": 289734656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046070210631895685, + "loss": 3.0029, + "theoretical_loss": 4.163329561597885, + "tokens_seen": 289800192 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606920762286861, + "loss": 2.892, + "theoretical_loss": 4.1632184873147295, + "tokens_seen": 289865728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046068204613841526, + "loss": 2.936, + "theoretical_loss": 4.1631074451714145, + "tokens_seen": 289931264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 171595, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1895124912261963, + "objective/train/theoretical_loss": 4.162996435151379, + "objective/train/tokens_used": 310456800, + "theoretical_loss": 4.162996435151379, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046067201604814445, + "loss": 3.0838, + "theoretical_loss": 4.162996435151379, + "tokens_seen": 289996800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606619859578736, + "loss": 2.8576, + "theoretical_loss": 4.1628854572380725, + "tokens_seen": 290062336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606519558676028, + "loss": 2.9221, + "theoretical_loss": 4.162774511414959, + "tokens_seen": 290127872 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460641925777332, + "loss": 2.9766, + "theoretical_loss": 4.162663597665513, + "tokens_seen": 290193408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606318956870612, + "loss": 3.1019, + "theoretical_loss": 4.162552715973222, + "tokens_seen": 290258944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046062186559679035, + "loss": 3.0226, + "theoretical_loss": 4.162441866321586, + "tokens_seen": 290324480 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004606118355065196, + "loss": 3.1766, + "theoretical_loss": 4.162331048694118, + "tokens_seen": 290390016 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046060180541624877, + "loss": 2.9799, + "theoretical_loss": 4.162220263074341, + "tokens_seen": 290455552 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046059177532597795, + "loss": 3.0867, + "theoretical_loss": 4.1621095094457905, + "tokens_seen": 290521088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046058174523570713, + "loss": 3.0066, + "theoretical_loss": 4.161998787792016, + "tokens_seen": 290586624 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605717151454363, + "loss": 3.0877, + "theoretical_loss": 4.161888098096579, + "tokens_seen": 290652160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605616850551655, + "loss": 2.8912, + "theoretical_loss": 4.161777440343051, + "tokens_seen": 290717696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605516549648947, + "loss": 3.0279, + "theoretical_loss": 4.161666814515017, + "tokens_seen": 290783232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046054162487462385, + "loss": 2.9034, + "theoretical_loss": 4.161556220596075, + "tokens_seen": 290848768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605315947843531, + "loss": 2.9769, + "theoretical_loss": 4.161445658569833, + "tokens_seen": 290914304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004605215646940822, + "loss": 3.0383, + "theoretical_loss": 4.161335128419914, + "tokens_seen": 290979840 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046051153460381145, + "loss": 2.9355, + "theoretical_loss": 4.16122463012995, + "tokens_seen": 291045376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046050150451354063, + "loss": 3.299, + "theoretical_loss": 4.161114163683587, + "tokens_seen": 291110912 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604914744232698, + "loss": 2.8822, + "theoretical_loss": 4.161003729064483, + "tokens_seen": 291176448 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460481444332999, + "loss": 3.0251, + "theoretical_loss": 4.160893326256307, + "tokens_seen": 291241984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046047141424272823, + "loss": 2.8968, + "theoretical_loss": 4.160782955242741, + "tokens_seen": 291307520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046046138415245736, + "loss": 3.0828, + "theoretical_loss": 4.160672616007479, + "tokens_seen": 291373056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604513540621866, + "loss": 3.2492, + "theoretical_loss": 4.160562308534226, + "tokens_seen": 291438592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604413239719157, + "loss": 2.7536, + "theoretical_loss": 4.160452032806701, + "tokens_seen": 291504128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046043129388164495, + "loss": 3.1393, + "theoretical_loss": 4.160341788808633, + "tokens_seen": 291569664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 172214, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0195140838623047, + "objective/train/theoretical_loss": 4.160231576523763, + "objective/train/tokens_used": 312095200, + "theoretical_loss": 4.160231576523763, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046042126379137414, + "loss": 2.9725, + "theoretical_loss": 4.160231576523763, + "tokens_seen": 291635200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604112337011033, + "loss": 3.0591, + "theoretical_loss": 4.160121395935847, + "tokens_seen": 291700736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004604012036108325, + "loss": 2.6618, + "theoretical_loss": 4.160011247028649, + "tokens_seen": 291766272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603911735205617, + "loss": 2.9955, + "theoretical_loss": 4.159901129785947, + "tokens_seen": 291831808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046038114343029086, + "loss": 2.9759, + "theoretical_loss": 4.159791044191532, + "tokens_seen": 291897344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603711133400201, + "loss": 2.9241, + "theoretical_loss": 4.1596809902292025, + "tokens_seen": 291962880 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603610832497492, + "loss": 3.0356, + "theoretical_loss": 4.159570967882775, + "tokens_seen": 292028416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046035105315947846, + "loss": 2.8435, + "theoretical_loss": 4.1594609771360735, + "tokens_seen": 292093952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603410230692076, + "loss": 3.1677, + "theoretical_loss": 4.159351017972936, + "tokens_seen": 292159488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603309929789368, + "loss": 2.7557, + "theoretical_loss": 4.159241090377211, + "tokens_seen": 292225024 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460320962888666, + "loss": 2.997, + "theoretical_loss": 4.15913119433276, + "tokens_seen": 292290560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004603109327983952, + "loss": 3.1299, + "theoretical_loss": 4.159021329823457, + "tokens_seen": 292356096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046030090270812436, + "loss": 2.9866, + "theoretical_loss": 4.158911496833184, + "tokens_seen": 292421632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602908726178536, + "loss": 2.8174, + "theoretical_loss": 4.15880169534584, + "tokens_seen": 292487168 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602808425275827, + "loss": 3.0642, + "theoretical_loss": 4.158691925345334, + "tokens_seen": 292552704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046027081243731196, + "loss": 2.9502, + "theoretical_loss": 4.158582186815584, + "tokens_seen": 292618240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602607823470411, + "loss": 2.957, + "theoretical_loss": 4.158472479740524, + "tokens_seen": 292683776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602507522567703, + "loss": 2.9731, + "theoretical_loss": 4.158362804104097, + "tokens_seen": 292749312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602407221664995, + "loss": 3.0535, + "theoretical_loss": 4.1582531598902595, + "tokens_seen": 292814848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602306920762287, + "loss": 2.985, + "theoretical_loss": 4.158143547082978, + "tokens_seen": 292880384 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046022066198595787, + "loss": 2.9562, + "theoretical_loss": 4.158033965666233, + "tokens_seen": 292945920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046021063189568705, + "loss": 2.8592, + "theoretical_loss": 4.157924415624014, + "tokens_seen": 293011456 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004602006018054163, + "loss": 2.9666, + "theoretical_loss": 4.157814896940324, + "tokens_seen": 293076992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046019057171514546, + "loss": 3.2116, + "theoretical_loss": 4.157705409599179, + "tokens_seen": 293142528 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046018054162487465, + "loss": 2.9664, + "theoretical_loss": 4.157595953584604, + "tokens_seen": 293208064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 173342, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8450427055358887, + "objective/train/theoretical_loss": 4.1574865288806375, + "objective/train/tokens_used": 313733600, + "theoretical_loss": 4.1574865288806375, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601705115346038, + "loss": 2.9126, + "theoretical_loss": 4.1574865288806375, + "tokens_seen": 293273600 + }, + { + "epoch": 0.09, + "learning_rate": 0.000460160481444333, + "loss": 2.7492, + "theoretical_loss": 4.157377135471328, + "tokens_seen": 293339136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601504513540622, + "loss": 2.8848, + "theoretical_loss": 4.157267773340738, + "tokens_seen": 293404672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601404212637914, + "loss": 3.0064, + "theoretical_loss": 4.15715844247294, + "tokens_seen": 293470208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046013039117352055, + "loss": 2.9549, + "theoretical_loss": 4.157049142852019, + "tokens_seen": 293535744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004601203610832498, + "loss": 2.9487, + "theoretical_loss": 4.15693987446207, + "tokens_seen": 293601280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046011033099297897, + "loss": 3.0182, + "theoretical_loss": 4.156830637287203, + "tokens_seen": 293666816 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046010030090270815, + "loss": 2.9087, + "theoretical_loss": 4.156721431311535, + "tokens_seen": 293732352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046009027081243733, + "loss": 3.1088, + "theoretical_loss": 4.1566122565192, + "tokens_seen": 293797888 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600802407221665, + "loss": 3.0616, + "theoretical_loss": 4.156503112894339, + "tokens_seen": 293863424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600702106318957, + "loss": 2.9094, + "theoretical_loss": 4.156394000421107, + "tokens_seen": 293928960 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046006018054162493, + "loss": 3.0927, + "theoretical_loss": 4.15628491908367, + "tokens_seen": 293994496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046005015045135405, + "loss": 3.1256, + "theoretical_loss": 4.156175868866205, + "tokens_seen": 294060032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600401203610833, + "loss": 3.0551, + "theoretical_loss": 4.156066849752902, + "tokens_seen": 294125568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004600300902708124, + "loss": 3.0325, + "theoretical_loss": 4.15595786172796, + "tokens_seen": 294191104 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046002006018054165, + "loss": 3.0107, + "theoretical_loss": 4.155848904775594, + "tokens_seen": 294256640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046001003009027083, + "loss": 2.9292, + "theoretical_loss": 4.155739978880025, + "tokens_seen": 294322176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00046, + "loss": 2.831, + "theoretical_loss": 4.155631084025489, + "tokens_seen": 294387712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599899699097292, + "loss": 3.1475, + "theoretical_loss": 4.155522220196234, + "tokens_seen": 294453248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045997993981945843, + "loss": 2.9871, + "theoretical_loss": 4.155413387376518, + "tokens_seen": 294518784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045996990972918756, + "loss": 3.1932, + "theoretical_loss": 4.155304585550609, + "tokens_seen": 294584320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599598796389168, + "loss": 2.9215, + "theoretical_loss": 4.15519581470279, + "tokens_seen": 294649856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599498495486459, + "loss": 2.9558, + "theoretical_loss": 4.155087074817353, + "tokens_seen": 294715392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045993981945837515, + "loss": 3.2962, + "theoretical_loss": 4.154978365878603, + "tokens_seen": 294780928 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045992978936810434, + "loss": 3.1586, + "theoretical_loss": 4.154869687870854, + "tokens_seen": 294846464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 173989, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.6182363033294678, + "objective/train/theoretical_loss": 4.1547610407784346, + "objective/train/tokens_used": 315372000, + "theoretical_loss": 4.1547610407784346, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599197592778335, + "loss": 3.244, + "theoretical_loss": 4.1547610407784346, + "tokens_seen": 294912000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004599097291875627, + "loss": 3.042, + "theoretical_loss": 4.154652424585683, + "tokens_seen": 294977536 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598996990972919, + "loss": 2.722, + "theoretical_loss": 4.154543839276948, + "tokens_seen": 295043072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045988966900702106, + "loss": 3.0362, + "theoretical_loss": 4.154435284836591, + "tokens_seen": 295108608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598796389167503, + "loss": 2.8414, + "theoretical_loss": 4.154326761248987, + "tokens_seen": 295174144 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598696088264794, + "loss": 2.858, + "theoretical_loss": 4.154218268498517, + "tokens_seen": 295239680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045985957873620866, + "loss": 2.9538, + "theoretical_loss": 4.154109806569577, + "tokens_seen": 295305216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598495486459378, + "loss": 3.0794, + "theoretical_loss": 4.154001375446576, + "tokens_seen": 295370752 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459839518555667, + "loss": 3.0819, + "theoretical_loss": 4.15389297511393, + "tokens_seen": 295436288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598294884653962, + "loss": 3.1095, + "theoretical_loss": 4.153784605556069, + "tokens_seen": 295501824 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004598194583751254, + "loss": 3.2757, + "theoretical_loss": 4.153676266757434, + "tokens_seen": 295567360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045980942828485456, + "loss": 2.7567, + "theoretical_loss": 4.153567958702476, + "tokens_seen": 295632896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597993981945838, + "loss": 3.0827, + "theoretical_loss": 4.153459681375661, + "tokens_seen": 295698432 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597893681043129, + "loss": 3.2228, + "theoretical_loss": 4.15335143476146, + "tokens_seen": 295763968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045977933801404216, + "loss": 2.9004, + "theoretical_loss": 4.153243218844362, + "tokens_seen": 295829504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597693079237713, + "loss": 2.9718, + "theoretical_loss": 4.153135033608863, + "tokens_seen": 295895040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597592778335005, + "loss": 2.9733, + "theoretical_loss": 4.153026879039471, + "tokens_seen": 295960576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597492477432297, + "loss": 2.9921, + "theoretical_loss": 4.152918755120708, + "tokens_seen": 296026112 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004597392176529589, + "loss": 2.9436, + "theoretical_loss": 4.152810661837102, + "tokens_seen": 296091648 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045972918756268807, + "loss": 3.1248, + "theoretical_loss": 4.152702599173197, + "tokens_seen": 296157184 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045971915747241725, + "loss": 3.1309, + "theoretical_loss": 4.152594567113546, + "tokens_seen": 296222720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045970912738214643, + "loss": 2.9432, + "theoretical_loss": 4.152486565642714, + "tokens_seen": 296288256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045969909729187566, + "loss": 3.2128, + "theoretical_loss": 4.152378594745276, + "tokens_seen": 296353792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596890672016048, + "loss": 3.153, + "theoretical_loss": 4.15227065440582, + "tokens_seen": 296419328 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459679037111334, + "loss": 3.1355, + "theoretical_loss": 4.152162744608944, + "tokens_seen": 296484864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 175144, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.142443895339966, + "objective/train/theoretical_loss": 4.152054865339256, + "objective/train/tokens_used": 317010400, + "theoretical_loss": 4.152054865339256, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045966900702106315, + "loss": 3.1106, + "theoretical_loss": 4.152054865339256, + "tokens_seen": 296550400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596589769307924, + "loss": 3.0485, + "theoretical_loss": 4.151947016581378, + "tokens_seen": 296615936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045964894684052157, + "loss": 3.0957, + "theoretical_loss": 4.151839198319942, + "tokens_seen": 296681472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045963891675025075, + "loss": 2.7138, + "theoretical_loss": 4.15173141053959, + "tokens_seen": 296747008 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045962888665997993, + "loss": 2.8906, + "theoretical_loss": 4.151623653224975, + "tokens_seen": 296812544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045961885656970917, + "loss": 2.9162, + "theoretical_loss": 4.151515926360764, + "tokens_seen": 296878080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004596088264794383, + "loss": 2.9349, + "theoretical_loss": 4.151408229931631, + "tokens_seen": 296943616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045959879638916753, + "loss": 3.0348, + "theoretical_loss": 4.151300563922266, + "tokens_seen": 297009152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045958876629889666, + "loss": 3.0585, + "theoretical_loss": 4.151192928317365, + "tokens_seen": 297074688 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595787362086259, + "loss": 2.8223, + "theoretical_loss": 4.151085323101638, + "tokens_seen": 297140224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595687061183551, + "loss": 2.8607, + "theoretical_loss": 4.150977748259805, + "tokens_seen": 297205760 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045955867602808425, + "loss": 2.8969, + "theoretical_loss": 4.1508702037765985, + "tokens_seen": 297271296 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045954864593781344, + "loss": 2.9414, + "theoretical_loss": 4.150762689636761, + "tokens_seen": 297336832 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595386158475426, + "loss": 3.0862, + "theoretical_loss": 4.150655205825045, + "tokens_seen": 297402368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004595285857572718, + "loss": 3.018, + "theoretical_loss": 4.150547752326216, + "tokens_seen": 297467904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045951855566700103, + "loss": 2.9801, + "theoretical_loss": 4.15044032912505, + "tokens_seen": 297533440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045950852557673016, + "loss": 3.0379, + "theoretical_loss": 4.150332936206332, + "tokens_seen": 297598976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594984954864594, + "loss": 3.249, + "theoretical_loss": 4.150225573554859, + "tokens_seen": 297664512 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594884653961885, + "loss": 3.0122, + "theoretical_loss": 4.150118241155443, + "tokens_seen": 297730048 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045947843530591776, + "loss": 3.2498, + "theoretical_loss": 4.1500109389929, + "tokens_seen": 297795584 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045946840521564694, + "loss": 2.7565, + "theoretical_loss": 4.1499036670520635, + "tokens_seen": 297861120 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594583751253761, + "loss": 3.0607, + "theoretical_loss": 4.149796425317772, + "tokens_seen": 297926656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045944834503510536, + "loss": 3.1681, + "theoretical_loss": 4.149689213774879, + "tokens_seen": 297992192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045943831494483454, + "loss": 3.1897, + "theoretical_loss": 4.149582032408249, + "tokens_seen": 298057728 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594282848545637, + "loss": 3.1223, + "theoretical_loss": 4.149474881202755, + "tokens_seen": 298123264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 176258, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.113511323928833, + "objective/train/theoretical_loss": 4.1493677601432815, + "objective/train/tokens_used": 318648800, + "theoretical_loss": 4.1493677601432815, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594182547642929, + "loss": 3.0056, + "theoretical_loss": 4.1493677601432815, + "tokens_seen": 298188800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004594082246740221, + "loss": 3.017, + "theoretical_loss": 4.149260669214725, + "tokens_seen": 298254336 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045939819458375126, + "loss": 3.0696, + "theoretical_loss": 4.149153608401994, + "tokens_seen": 298319872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593881644934805, + "loss": 2.9312, + "theoretical_loss": 4.149046577690003, + "tokens_seen": 298385408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593781344032096, + "loss": 2.979, + "theoretical_loss": 4.148939577063683, + "tokens_seen": 298450944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045936810431293886, + "loss": 2.7, + "theoretical_loss": 4.148832606507974, + "tokens_seen": 298516480 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459358074222668, + "loss": 2.9176, + "theoretical_loss": 4.148725666007824, + "tokens_seen": 298582016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593480441323972, + "loss": 2.9862, + "theoretical_loss": 4.1486187555481955, + "tokens_seen": 298647552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593380140421264, + "loss": 2.7821, + "theoretical_loss": 4.14851187511406, + "tokens_seen": 298713088 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004593279839518556, + "loss": 3.0391, + "theoretical_loss": 4.148405024690402, + "tokens_seen": 298778624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045931795386158476, + "loss": 2.9083, + "theoretical_loss": 4.148298204262213, + "tokens_seen": 298844160 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459307923771314, + "loss": 2.8753, + "theoretical_loss": 4.148191413814497, + "tokens_seen": 298909696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592978936810431, + "loss": 2.9916, + "theoretical_loss": 4.148084653332271, + "tokens_seen": 298975232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045928786359077236, + "loss": 3.0754, + "theoretical_loss": 4.14797792280056, + "tokens_seen": 299040768 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592778335005015, + "loss": 3.0386, + "theoretical_loss": 4.147871222204401, + "tokens_seen": 299106304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592678034102307, + "loss": 2.9645, + "theoretical_loss": 4.147764551528841, + "tokens_seen": 299171840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592577733199599, + "loss": 2.7558, + "theoretical_loss": 4.147657910758938, + "tokens_seen": 299237376 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004592477432296891, + "loss": 2.7373, + "theoretical_loss": 4.147551299879762, + "tokens_seen": 299302912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045923771313941827, + "loss": 2.8164, + "theoretical_loss": 4.147444718876391, + "tokens_seen": 299368448 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045922768304914745, + "loss": 2.8048, + "theoretical_loss": 4.147338167733917, + "tokens_seen": 299433984 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045921765295887663, + "loss": 3.0629, + "theoretical_loss": 4.14723164643744, + "tokens_seen": 299499520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045920762286860586, + "loss": 2.948, + "theoretical_loss": 4.1471251549720725, + "tokens_seen": 299565056 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459197592778335, + "loss": 2.9864, + "theoretical_loss": 4.147018693322936, + "tokens_seen": 299630592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591875626880642, + "loss": 2.9198, + "theoretical_loss": 4.146912261475165, + "tokens_seen": 299696128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045917753259779335, + "loss": 3.034, + "theoretical_loss": 4.1468058594139015, + "tokens_seen": 299761664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 176682, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.946582078933716, + "objective/train/theoretical_loss": 4.146699487124301, + "objective/train/tokens_used": 320287200, + "theoretical_loss": 4.146699487124301, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591675025075226, + "loss": 3.0244, + "theoretical_loss": 4.146699487124301, + "tokens_seen": 299827200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045915747241725177, + "loss": 3.0344, + "theoretical_loss": 4.146593144591527, + "tokens_seen": 299892736 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045914744232698095, + "loss": 3.1026, + "theoretical_loss": 4.146486831800758, + "tokens_seen": 299958272 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045913741223671013, + "loss": 3.005, + "theoretical_loss": 4.146380548737179, + "tokens_seen": 300023808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045912738214643937, + "loss": 3.1644, + "theoretical_loss": 4.146274295385986, + "tokens_seen": 300089344 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004591173520561685, + "loss": 2.8718, + "theoretical_loss": 4.146168071732388, + "tokens_seen": 300154880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045910732196589773, + "loss": 3.0563, + "theoretical_loss": 4.146061877761602, + "tokens_seen": 300220416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045909729187562686, + "loss": 3.226, + "theoretical_loss": 4.145955713458856, + "tokens_seen": 300285952 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590872617853561, + "loss": 3.0286, + "theoretical_loss": 4.145849578809392, + "tokens_seen": 300351488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590772316950853, + "loss": 2.9362, + "theoretical_loss": 4.145743473798457, + "tokens_seen": 300417024 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045906720160481445, + "loss": 3.3005, + "theoretical_loss": 4.145637398411313, + "tokens_seen": 300482560 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045905717151454364, + "loss": 3.1683, + "theoretical_loss": 4.145531352633229, + "tokens_seen": 300548096 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590471414242728, + "loss": 2.8494, + "theoretical_loss": 4.14542533644949, + "tokens_seen": 300613632 + }, + { + "epoch": 0.09, + "learning_rate": 0.000459037111334002, + "loss": 2.7638, + "theoretical_loss": 4.145319349845384, + "tokens_seen": 300679168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045902708124373123, + "loss": 3.1229, + "theoretical_loss": 4.145213392806217, + "tokens_seen": 300744704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045901705115346036, + "loss": 2.8249, + "theoretical_loss": 4.145107465317299, + "tokens_seen": 300810240 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004590070210631896, + "loss": 2.7708, + "theoretical_loss": 4.1450015673639555, + "tokens_seen": 300875776 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589969909729187, + "loss": 2.9347, + "theoretical_loss": 4.144895698931521, + "tokens_seen": 300941312 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045898696088264796, + "loss": 3.1446, + "theoretical_loss": 4.144789860005337, + "tokens_seen": 301006848 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045897693079237714, + "loss": 2.7626, + "theoretical_loss": 4.14468405057076, + "tokens_seen": 301072384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589669007021063, + "loss": 2.9021, + "theoretical_loss": 4.144578270613157, + "tokens_seen": 301137920 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589568706118355, + "loss": 2.9562, + "theoretical_loss": 4.144472520117901, + "tokens_seen": 301203456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045894684052156474, + "loss": 3.0293, + "theoretical_loss": 4.14436679907038, + "tokens_seen": 301268992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045893681043129386, + "loss": 2.9459, + "theoretical_loss": 4.144261107455991, + "tokens_seen": 301334528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589267803410231, + "loss": 2.9452, + "theoretical_loss": 4.1441554452601395, + "tokens_seen": 301400064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 177896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9508297443389893, + "objective/train/theoretical_loss": 4.1440498124682446, + "objective/train/tokens_used": 321925600, + "theoretical_loss": 4.1440498124682446, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004589167502507522, + "loss": 3.0381, + "theoretical_loss": 4.1440498124682446, + "tokens_seen": 301465600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045890672016048146, + "loss": 3.0254, + "theoretical_loss": 4.143944209065733, + "tokens_seen": 301531136 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045889669007021064, + "loss": 2.948, + "theoretical_loss": 4.143838635038043, + "tokens_seen": 301596672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588866599799398, + "loss": 3.035, + "theoretical_loss": 4.143733090370624, + "tokens_seen": 301662208 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458876629889669, + "loss": 2.9586, + "theoretical_loss": 4.143627575048935, + "tokens_seen": 301727744 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588665997993982, + "loss": 2.904, + "theoretical_loss": 4.143522089058446, + "tokens_seen": 301793280 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045885656970912737, + "loss": 2.9791, + "theoretical_loss": 4.143416632384635, + "tokens_seen": 301858816 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588465396188566, + "loss": 2.9168, + "theoretical_loss": 4.143311205012992, + "tokens_seen": 301924352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045883650952858573, + "loss": 2.9416, + "theoretical_loss": 4.14320580692902, + "tokens_seen": 301989888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045882647943831496, + "loss": 3.2419, + "theoretical_loss": 4.143100438118227, + "tokens_seen": 302055424 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045881644934804415, + "loss": 3.0354, + "theoretical_loss": 4.142995098566135, + "tokens_seen": 302120960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004588064192577733, + "loss": 2.891, + "theoretical_loss": 4.142889788258275, + "tokens_seen": 302186496 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587963891675025, + "loss": 3.0476, + "theoretical_loss": 4.142784507180189, + "tokens_seen": 302252032 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587863590772317, + "loss": 2.9312, + "theoretical_loss": 4.14267925531743, + "tokens_seen": 302317568 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045877632898696087, + "loss": 3.0632, + "theoretical_loss": 4.142574032655558, + "tokens_seen": 302383104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587662988966901, + "loss": 3.0175, + "theoretical_loss": 4.142468839180148, + "tokens_seen": 302448640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045875626880641923, + "loss": 3.1759, + "theoretical_loss": 4.14236367487678, + "tokens_seen": 302514176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045874623871614847, + "loss": 3.0104, + "theoretical_loss": 4.142258539731047, + "tokens_seen": 302579712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587362086258776, + "loss": 2.8409, + "theoretical_loss": 4.142153433728556, + "tokens_seen": 302645248 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045872617853560683, + "loss": 3.0343, + "theoretical_loss": 4.142048356854915, + "tokens_seen": 302710784 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458716148445336, + "loss": 3.1169, + "theoretical_loss": 4.141943309095752, + "tokens_seen": 302776320 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004587061183550652, + "loss": 2.7164, + "theoretical_loss": 4.141838290436699, + "tokens_seen": 302841856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586960882647944, + "loss": 3.1328, + "theoretical_loss": 4.1417333008633985, + "tokens_seen": 302907392 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045868605817452355, + "loss": 2.9251, + "theoretical_loss": 4.141628340361508, + "tokens_seen": 302972928 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586760280842528, + "loss": 3.0941, + "theoretical_loss": 4.141523408916689, + "tokens_seen": 303038464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 178251, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.023979425430298, + "objective/train/theoretical_loss": 4.141418506514618, + "objective/train/tokens_used": 323564000, + "theoretical_loss": 4.141418506514618, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045866599799398197, + "loss": 3.0323, + "theoretical_loss": 4.141418506514618, + "tokens_seen": 303104000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045865596790371115, + "loss": 2.966, + "theoretical_loss": 4.141313633140978, + "tokens_seen": 303169536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045864593781344033, + "loss": 3.0848, + "theoretical_loss": 4.141208788781466, + "tokens_seen": 303235072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045863590772316957, + "loss": 3.0573, + "theoretical_loss": 4.141103973421784, + "tokens_seen": 303300608 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004586258776328987, + "loss": 3.0507, + "theoretical_loss": 4.14099918704765, + "tokens_seen": 303366144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045861584754262793, + "loss": 2.9278, + "theoretical_loss": 4.140894429644788, + "tokens_seen": 303431680 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045860581745235706, + "loss": 3.1005, + "theoretical_loss": 4.1407897011989325, + "tokens_seen": 303497216 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585957873620863, + "loss": 3.0779, + "theoretical_loss": 4.1406850016958305, + "tokens_seen": 303562752 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585857572718155, + "loss": 2.8597, + "theoretical_loss": 4.140580331121236, + "tokens_seen": 303628288 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045857572718154465, + "loss": 2.8306, + "theoretical_loss": 4.140475689460916, + "tokens_seen": 303693824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045856569709127384, + "loss": 3.0461, + "theoretical_loss": 4.140371076700646, + "tokens_seen": 303759360 + }, + { + "epoch": 0.09, + "learning_rate": 0.000458555667001003, + "loss": 3.0263, + "theoretical_loss": 4.140266492826211, + "tokens_seen": 303824896 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585456369107322, + "loss": 3.0801, + "theoretical_loss": 4.140161937823407, + "tokens_seen": 303890432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045853560682046143, + "loss": 2.9155, + "theoretical_loss": 4.140057411678041, + "tokens_seen": 303955968 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045852557673019056, + "loss": 2.8495, + "theoretical_loss": 4.139952914375927, + "tokens_seen": 304021504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585155466399198, + "loss": 2.9827, + "theoretical_loss": 4.139848445902892, + "tokens_seen": 304087040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004585055165496489, + "loss": 2.9606, + "theoretical_loss": 4.139744006244772, + "tokens_seen": 304152576 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045849548645937816, + "loss": 2.8089, + "theoretical_loss": 4.139639595387413, + "tokens_seen": 304218112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045848545636910734, + "loss": 3.0314, + "theoretical_loss": 4.139535213316671, + "tokens_seen": 304283648 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584754262788365, + "loss": 2.972, + "theoretical_loss": 4.139430860018411, + "tokens_seen": 304349184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584653961885657, + "loss": 3.0738, + "theoretical_loss": 4.13932653547851, + "tokens_seen": 304414720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045845536609829494, + "loss": 2.8192, + "theoretical_loss": 4.139222239682853, + "tokens_seen": 304480256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045844533600802406, + "loss": 2.805, + "theoretical_loss": 4.139117972617337, + "tokens_seen": 304545792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584353059177533, + "loss": 2.8678, + "theoretical_loss": 4.139013734267866, + "tokens_seen": 304611328 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004584252758274824, + "loss": 2.9425, + "theoretical_loss": 4.138909524620358, + "tokens_seen": 304676864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 179536, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0331430435180664, + "objective/train/theoretical_loss": 4.138805343660737, + "objective/train/tokens_used": 325202400, + "theoretical_loss": 4.138805343660737, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045841524573721166, + "loss": 2.9569, + "theoretical_loss": 4.138805343660737, + "tokens_seen": 304742400 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045840521564694084, + "loss": 3.0136, + "theoretical_loss": 4.13870119137494, + "tokens_seen": 304807936 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045839518555667, + "loss": 2.7669, + "theoretical_loss": 4.138597067748911, + "tokens_seen": 304873472 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583851554663992, + "loss": 2.9702, + "theoretical_loss": 4.138492972768606, + "tokens_seen": 304939008 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583751253761284, + "loss": 3.1139, + "theoretical_loss": 4.138388906419991, + "tokens_seen": 305004544 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045836509528585757, + "loss": 2.9143, + "theoretical_loss": 4.138284868689041, + "tokens_seen": 305070080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583550651955868, + "loss": 2.7962, + "theoretical_loss": 4.13818085956174, + "tokens_seen": 305135616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045834503510531593, + "loss": 3.0717, + "theoretical_loss": 4.1380768790240845, + "tokens_seen": 305201152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045833500501504516, + "loss": 2.8641, + "theoretical_loss": 4.137972927062078, + "tokens_seen": 305266688 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045832497492477435, + "loss": 2.9519, + "theoretical_loss": 4.137869003661737, + "tokens_seen": 305332224 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583149448345035, + "loss": 3.1127, + "theoretical_loss": 4.137765108809084, + "tokens_seen": 305397760 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004583049147442327, + "loss": 2.9791, + "theoretical_loss": 4.1376612424901555, + "tokens_seen": 305463296 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582948846539619, + "loss": 2.9104, + "theoretical_loss": 4.137557404690993, + "tokens_seen": 305528832 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045828485456369107, + "loss": 3.0647, + "theoretical_loss": 4.137453595397653, + "tokens_seen": 305594368 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582748244734203, + "loss": 2.9847, + "theoretical_loss": 4.137349814596198, + "tokens_seen": 305659904 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045826479438314943, + "loss": 3.0683, + "theoretical_loss": 4.137246062272702, + "tokens_seen": 305725440 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045825476429287867, + "loss": 2.9846, + "theoretical_loss": 4.137142338413248, + "tokens_seen": 305790976 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582447342026078, + "loss": 2.7466, + "theoretical_loss": 4.137038643003931, + "tokens_seen": 305856512 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045823470411233703, + "loss": 2.9283, + "theoretical_loss": 4.136934976030852, + "tokens_seen": 305922048 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582246740220662, + "loss": 3.0245, + "theoretical_loss": 4.136831337480125, + "tokens_seen": 305987584 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004582146439317954, + "loss": 2.8523, + "theoretical_loss": 4.13672772733787, + "tokens_seen": 306053120 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045820461384152457, + "loss": 2.8644, + "theoretical_loss": 4.136624145590222, + "tokens_seen": 306118656 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045819458375125375, + "loss": 3.1607, + "theoretical_loss": 4.136520592223322, + "tokens_seen": 306184192 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045818455366098294, + "loss": 2.9101, + "theoretical_loss": 4.136417067223322, + "tokens_seen": 306249728 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045817452357071217, + "loss": 3.1469, + "theoretical_loss": 4.136313570576383, + "tokens_seen": 306315264 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 180132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.187363624572754, + "objective/train/theoretical_loss": 4.136210102268676, + "objective/train/tokens_used": 326840800, + "theoretical_loss": 4.136210102268676, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581644934804413, + "loss": 3.0491, + "theoretical_loss": 4.136210102268676, + "tokens_seen": 306380800 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045815446339017053, + "loss": 3.2312, + "theoretical_loss": 4.136106662286383, + "tokens_seen": 306446336 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581444332998997, + "loss": 3.1742, + "theoretical_loss": 4.136003250615693, + "tokens_seen": 306511872 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581344032096289, + "loss": 2.983, + "theoretical_loss": 4.135899867242808, + "tokens_seen": 306577408 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004581243731193581, + "loss": 3.1178, + "theoretical_loss": 4.135796512153934, + "tokens_seen": 306642944 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045811434302908726, + "loss": 2.8261, + "theoretical_loss": 4.135693185335295, + "tokens_seen": 306708480 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045810431293881644, + "loss": 3.001, + "theoretical_loss": 4.135589886773119, + "tokens_seen": 306774016 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580942828485457, + "loss": 2.9546, + "theoretical_loss": 4.135486616453644, + "tokens_seen": 306839552 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580842527582748, + "loss": 3.0585, + "theoretical_loss": 4.135383374363119, + "tokens_seen": 306905088 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045807422266800404, + "loss": 3.023, + "theoretical_loss": 4.1352801604878024, + "tokens_seen": 306970624 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045806419257773316, + "loss": 2.954, + "theoretical_loss": 4.135176974813961, + "tokens_seen": 307036160 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580541624874624, + "loss": 3.259, + "theoretical_loss": 4.135073817327872, + "tokens_seen": 307101696 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580441323971916, + "loss": 3.053, + "theoretical_loss": 4.134970688015824, + "tokens_seen": 307167232 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045803410230692076, + "loss": 2.8994, + "theoretical_loss": 4.134867586864112, + "tokens_seen": 307232768 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045802407221664994, + "loss": 3.019, + "theoretical_loss": 4.134764513859043, + "tokens_seen": 307298304 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580140421263791, + "loss": 2.7869, + "theoretical_loss": 4.134661468986932, + "tokens_seen": 307363840 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004580040120361083, + "loss": 2.8573, + "theoretical_loss": 4.1345584522341055, + "tokens_seen": 307429376 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045799398194583754, + "loss": 2.8166, + "theoretical_loss": 4.134455463586898, + "tokens_seen": 307494912 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045798395185556667, + "loss": 2.6522, + "theoretical_loss": 4.134352503031652, + "tokens_seen": 307560448 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579739217652959, + "loss": 3.0032, + "theoretical_loss": 4.134249570554724, + "tokens_seen": 307625984 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579638916750251, + "loss": 2.8976, + "theoretical_loss": 4.134146666142476, + "tokens_seen": 307691520 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045795386158475426, + "loss": 2.7396, + "theoretical_loss": 4.134043789781281, + "tokens_seen": 307757056 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579438314944835, + "loss": 3.0013, + "theoretical_loss": 4.1339409414575226, + "tokens_seen": 307822592 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579338014042126, + "loss": 2.9292, + "theoretical_loss": 4.133838121157592, + "tokens_seen": 307888128 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045792377131394186, + "loss": 3.2311, + "theoretical_loss": 4.133735328867892, + "tokens_seen": 307953664 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 181325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.002859354019165, + "objective/train/theoretical_loss": 4.133632564574831, + "objective/train/tokens_used": 328479200, + "theoretical_loss": 4.133632564574831, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045791374122367104, + "loss": 2.918, + "theoretical_loss": 4.133632564574831, + "tokens_seen": 308019200 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004579037111334002, + "loss": 2.8607, + "theoretical_loss": 4.133529828264831, + "tokens_seen": 308084736 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578936810431294, + "loss": 2.9817, + "theoretical_loss": 4.133427119924323, + "tokens_seen": 308150272 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578836509528586, + "loss": 3.1387, + "theoretical_loss": 4.133324439539745, + "tokens_seen": 308215808 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045787362086258777, + "loss": 3.2184, + "theoretical_loss": 4.133221787097546, + "tokens_seen": 308281344 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457863590772317, + "loss": 2.8897, + "theoretical_loss": 4.133119162584185, + "tokens_seen": 308346880 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045785356068204613, + "loss": 3.0767, + "theoretical_loss": 4.1330165659861295, + "tokens_seen": 308412416 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045784353059177536, + "loss": 2.9042, + "theoretical_loss": 4.132913997289856, + "tokens_seen": 308477952 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045783350050150455, + "loss": 2.9545, + "theoretical_loss": 4.1328114564818526, + "tokens_seen": 308543488 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578234704112337, + "loss": 2.8544, + "theoretical_loss": 4.132708943548614, + "tokens_seen": 308609024 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578134403209629, + "loss": 3.1091, + "theoretical_loss": 4.132606458476647, + "tokens_seen": 308674560 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004578034102306921, + "loss": 2.9141, + "theoretical_loss": 4.132504001252465, + "tokens_seen": 308740096 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045779338014042127, + "loss": 2.9562, + "theoretical_loss": 4.132401571862593, + "tokens_seen": 308805632 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577833500501505, + "loss": 2.9368, + "theoretical_loss": 4.132299170293564, + "tokens_seen": 308871168 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045777331995987963, + "loss": 2.8221, + "theoretical_loss": 4.132196796531922, + "tokens_seen": 308936704 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045776328986960887, + "loss": 2.9433, + "theoretical_loss": 4.132094450564218, + "tokens_seen": 309002240 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457753259779338, + "loss": 2.8562, + "theoretical_loss": 4.1319921323770155, + "tokens_seen": 309067776 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045774322968906723, + "loss": 3.0209, + "theoretical_loss": 4.131889841956884, + "tokens_seen": 309133312 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577331995987964, + "loss": 3.009, + "theoretical_loss": 4.131787579290405, + "tokens_seen": 309198848 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577231695085256, + "loss": 2.7863, + "theoretical_loss": 4.1316853443641675, + "tokens_seen": 309264384 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004577131394182548, + "loss": 3.0576, + "theoretical_loss": 4.13158313716477, + "tokens_seen": 309329920 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045770310932798395, + "loss": 2.9071, + "theoretical_loss": 4.131480957678823, + "tokens_seen": 309395456 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045769307923771314, + "loss": 2.9949, + "theoretical_loss": 4.1313788058929415, + "tokens_seen": 309460992 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045768304914744237, + "loss": 2.8319, + "theoretical_loss": 4.131276681793754, + "tokens_seen": 309526528 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576730190571715, + "loss": 2.7952, + "theoretical_loss": 4.131174585367896, + "tokens_seen": 309592064 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 182674, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9335567951202393, + "objective/train/theoretical_loss": 4.131072516602014, + "objective/train/tokens_used": 330117600, + "theoretical_loss": 4.131072516602014, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045766298896690073, + "loss": 2.9291, + "theoretical_loss": 4.131072516602014, + "tokens_seen": 309657600 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576529588766299, + "loss": 2.8759, + "theoretical_loss": 4.130970475482761, + "tokens_seen": 309723136 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576429287863591, + "loss": 2.7693, + "theoretical_loss": 4.130868461996803, + "tokens_seen": 309788672 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576328986960883, + "loss": 3.2834, + "theoretical_loss": 4.1307664761308125, + "tokens_seen": 309854208 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045762286860581746, + "loss": 2.7814, + "theoretical_loss": 4.130664517871471, + "tokens_seen": 309919744 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045761283851554664, + "loss": 3.364, + "theoretical_loss": 4.130562587205473, + "tokens_seen": 309985280 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004576028084252759, + "loss": 2.8317, + "theoretical_loss": 4.130460684119516, + "tokens_seen": 310050816 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457592778335005, + "loss": 2.9605, + "theoretical_loss": 4.130358808600313, + "tokens_seen": 310116352 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045758274824473424, + "loss": 3.1335, + "theoretical_loss": 4.130256960634582, + "tokens_seen": 310181888 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045757271815446336, + "loss": 3.0687, + "theoretical_loss": 4.130155140209052, + "tokens_seen": 310247424 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575626880641926, + "loss": 2.9797, + "theoretical_loss": 4.130053347310461, + "tokens_seen": 310312960 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575526579739218, + "loss": 3.294, + "theoretical_loss": 4.129951581925555, + "tokens_seen": 310378496 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045754262788365096, + "loss": 2.9568, + "theoretical_loss": 4.129849844041091, + "tokens_seen": 310444032 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045753259779338014, + "loss": 2.941, + "theoretical_loss": 4.129748133643835, + "tokens_seen": 310509568 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575225677031093, + "loss": 3.0219, + "theoretical_loss": 4.129646450720561, + "tokens_seen": 310575104 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004575125376128385, + "loss": 2.9852, + "theoretical_loss": 4.129544795258052, + "tokens_seen": 310640640 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045750250752256774, + "loss": 2.8331, + "theoretical_loss": 4.129443167243101, + "tokens_seen": 310706176 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045749247743229687, + "loss": 3.061, + "theoretical_loss": 4.129341566662511, + "tokens_seen": 310771712 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574824473420261, + "loss": 2.967, + "theoretical_loss": 4.1292399935030915, + "tokens_seen": 310837248 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574724172517553, + "loss": 3.0105, + "theoretical_loss": 4.129138447751664, + "tokens_seen": 310902784 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045746238716148446, + "loss": 2.9289, + "theoretical_loss": 4.129036929395058, + "tokens_seen": 310968320 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045745235707121364, + "loss": 3.0223, + "theoretical_loss": 4.12893543842011, + "tokens_seen": 311033856 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574423269809428, + "loss": 2.9751, + "theoretical_loss": 4.12883397481367, + "tokens_seen": 311099392 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457432296890672, + "loss": 2.8895, + "theoretical_loss": 4.128732538562592, + "tokens_seen": 311164928 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045742226680040124, + "loss": 2.7377, + "theoretical_loss": 4.128631129653744, + "tokens_seen": 311230464 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 183229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0134546756744385, + "objective/train/theoretical_loss": 4.128529748073999, + "objective/train/tokens_used": 331756000, + "theoretical_loss": 4.128529748073999, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045741223671013037, + "loss": 2.6315, + "theoretical_loss": 4.128529748073999, + "tokens_seen": 311296000 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004574022066198596, + "loss": 3.0727, + "theoretical_loss": 4.128428393810242, + "tokens_seen": 311361536 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045739217652958873, + "loss": 3.0927, + "theoretical_loss": 4.128327066849365, + "tokens_seen": 311427072 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045738214643931797, + "loss": 2.984, + "theoretical_loss": 4.12822576717827, + "tokens_seen": 311492608 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045737211634904715, + "loss": 3.1219, + "theoretical_loss": 4.128124494783869, + "tokens_seen": 311558144 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045736208625877633, + "loss": 2.8423, + "theoretical_loss": 4.12802324965308, + "tokens_seen": 311623680 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573520561685055, + "loss": 3.034, + "theoretical_loss": 4.127922031772834, + "tokens_seen": 311689216 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045734202607823475, + "loss": 3.0936, + "theoretical_loss": 4.127820841130068, + "tokens_seen": 311754752 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045733199598796387, + "loss": 2.8057, + "theoretical_loss": 4.127719677711729, + "tokens_seen": 311820288 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004573219658976931, + "loss": 2.8598, + "theoretical_loss": 4.1276185415047735, + "tokens_seen": 311885824 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045731193580742223, + "loss": 2.8944, + "theoretical_loss": 4.127517432496166, + "tokens_seen": 311951360 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045730190571715147, + "loss": 2.9157, + "theoretical_loss": 4.127416350672879, + "tokens_seen": 312016896 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045729187562688065, + "loss": 2.938, + "theoretical_loss": 4.1273152960219, + "tokens_seen": 312082432 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045728184553660983, + "loss": 3.0826, + "theoretical_loss": 4.127214268530216, + "tokens_seen": 312147968 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457271815446339, + "loss": 3.0073, + "theoretical_loss": 4.12711326818483, + "tokens_seen": 312213504 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572617853560682, + "loss": 3.0567, + "theoretical_loss": 4.127012294972753, + "tokens_seen": 312279040 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572517552657974, + "loss": 3.169, + "theoretical_loss": 4.126911348881, + "tokens_seen": 312344576 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572417251755266, + "loss": 2.9469, + "theoretical_loss": 4.126810429896603, + "tokens_seen": 312410112 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045723169508525574, + "loss": 2.9813, + "theoretical_loss": 4.126709538006596, + "tokens_seen": 312475648 + }, + { + "epoch": 0.09, + "learning_rate": 0.000457221664994985, + "loss": 2.934, + "theoretical_loss": 4.1266086731980245, + "tokens_seen": 312541184 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004572116349047141, + "loss": 3.2086, + "theoretical_loss": 4.126507835457945, + "tokens_seen": 312606720 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045720160481444334, + "loss": 2.7577, + "theoretical_loss": 4.126407024773418, + "tokens_seen": 312672256 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045719157472417257, + "loss": 2.9729, + "theoretical_loss": 4.126306241131519, + "tokens_seen": 312737792 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571815446339017, + "loss": 2.9627, + "theoretical_loss": 4.126205484519327, + "tokens_seen": 312803328 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045717151454363093, + "loss": 3.0685, + "theoretical_loss": 4.126104754923931, + "tokens_seen": 312868864 + }, + { + "epoch": 0.09, + "objective/train/docs_used": 184180, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2469050884246826, + "objective/train/theoretical_loss": 4.126004052332432, + "objective/train/tokens_used": 333394400, + "theoretical_loss": 4.126004052332432, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571614844533601, + "loss": 3.0363, + "theoretical_loss": 4.126004052332432, + "tokens_seen": 312934400 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571514543630893, + "loss": 2.9679, + "theoretical_loss": 4.125903376731936, + "tokens_seen": 312999936 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571414242728185, + "loss": 2.8487, + "theoretical_loss": 4.125802728109562, + "tokens_seen": 313065472 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045713139418254766, + "loss": 2.8367, + "theoretical_loss": 4.125702106452433, + "tokens_seen": 313131008 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045712136409227684, + "loss": 2.9977, + "theoretical_loss": 4.125601511747685, + "tokens_seen": 313196544 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571113340020061, + "loss": 3.0608, + "theoretical_loss": 4.125500943982459, + "tokens_seen": 313262080 + }, + { + "epoch": 0.09, + "learning_rate": 0.0004571013039117352, + "loss": 2.8297, + "theoretical_loss": 4.125400403143908, + "tokens_seen": 313327616 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045709127382146444, + "loss": 3.052, + "theoretical_loss": 4.125299889219194, + "tokens_seen": 313393152 + }, + { + "epoch": 0.09, + "learning_rate": 0.00045708124373119356, + "loss": 3.0677, + "theoretical_loss": 4.125199402195486, + "tokens_seen": 313458688 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570712136409228, + "loss": 2.9362, + "theoretical_loss": 4.12509894205996, + "tokens_seen": 313524224 + }, + { + "epoch": 0.1, + "learning_rate": 0.000457061183550652, + "loss": 2.9709, + "theoretical_loss": 4.124998508799806, + "tokens_seen": 313589760 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045705115346038116, + "loss": 3.04, + "theoretical_loss": 4.124898102402218, + "tokens_seen": 313655296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045704112337011034, + "loss": 3.0373, + "theoretical_loss": 4.124797722854401, + "tokens_seen": 313720832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570310932798395, + "loss": 2.8581, + "theoretical_loss": 4.12469737014357, + "tokens_seen": 313786368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004570210631895687, + "loss": 3.0926, + "theoretical_loss": 4.124597044256945, + "tokens_seen": 313851904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045701103309929794, + "loss": 3.0979, + "theoretical_loss": 4.124496745181759, + "tokens_seen": 313917440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045700100300902707, + "loss": 2.9877, + "theoretical_loss": 4.12439647290525, + "tokens_seen": 313982976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569909729187563, + "loss": 3.2337, + "theoretical_loss": 4.124296227414667, + "tokens_seen": 314048512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569809428284855, + "loss": 2.9723, + "theoretical_loss": 4.124196008697267, + "tokens_seen": 314114048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045697091273821466, + "loss": 3.2751, + "theoretical_loss": 4.124095816740317, + "tokens_seen": 314179584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045696088264794384, + "loss": 2.9168, + "theoretical_loss": 4.123995651531089, + "tokens_seen": 314245120 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456950852557673, + "loss": 2.9351, + "theoretical_loss": 4.123895513056869, + "tokens_seen": 314310656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569408224674022, + "loss": 3.0289, + "theoretical_loss": 4.123795401304948, + "tokens_seen": 314376192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045693079237713144, + "loss": 2.9276, + "theoretical_loss": 4.123695316262626, + "tokens_seen": 314441728 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045692076228686057, + "loss": 3.3239, + "theoretical_loss": 4.123595257917214, + "tokens_seen": 314507264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 184828, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.897738218307495, + "objective/train/theoretical_loss": 4.123495226256027, + "objective/train/tokens_used": 335032800, + "theoretical_loss": 4.123495226256027, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004569107321965898, + "loss": 2.8664, + "theoretical_loss": 4.123495226256027, + "tokens_seen": 314572800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045690070210631893, + "loss": 2.9238, + "theoretical_loss": 4.123395221266396, + "tokens_seen": 314638336 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045689067201604817, + "loss": 3.0188, + "theoretical_loss": 4.123295242935653, + "tokens_seen": 314703872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045688064192577735, + "loss": 2.9396, + "theoretical_loss": 4.123195291251142, + "tokens_seen": 314769408 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045687061183550653, + "loss": 3.1508, + "theoretical_loss": 4.123095366200217, + "tokens_seen": 314834944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568605817452357, + "loss": 2.9989, + "theoretical_loss": 4.122995467770239, + "tokens_seen": 314900480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045685055165496495, + "loss": 2.9205, + "theoretical_loss": 4.122895595948577, + "tokens_seen": 314966016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045684052156469407, + "loss": 2.8574, + "theoretical_loss": 4.122795750722612, + "tokens_seen": 315031552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004568304914744233, + "loss": 2.7756, + "theoretical_loss": 4.122695932079728, + "tokens_seen": 315097088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045682046138415243, + "loss": 3.1906, + "theoretical_loss": 4.122596140007323, + "tokens_seen": 315162624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045681043129388167, + "loss": 2.9296, + "theoretical_loss": 4.122496374492799, + "tokens_seen": 315228160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045680040120361085, + "loss": 2.6739, + "theoretical_loss": 4.122396635523571, + "tokens_seen": 315293696 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045679037111334003, + "loss": 2.8686, + "theoretical_loss": 4.12229692308706, + "tokens_seen": 315359232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567803410230692, + "loss": 3.2873, + "theoretical_loss": 4.122197237170695, + "tokens_seen": 315424768 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567703109327984, + "loss": 3.0861, + "theoretical_loss": 4.122097577761917, + "tokens_seen": 315490304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567602808425276, + "loss": 3.1712, + "theoretical_loss": 4.121997944848171, + "tokens_seen": 315555840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567502507522568, + "loss": 3.1188, + "theoretical_loss": 4.121898338416912, + "tokens_seen": 315621376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045674022066198594, + "loss": 2.9156, + "theoretical_loss": 4.121798758455608, + "tokens_seen": 315686912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567301905717152, + "loss": 2.9989, + "theoretical_loss": 4.121699204951728, + "tokens_seen": 315752448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567201604814443, + "loss": 2.6808, + "theoretical_loss": 4.121599677892755, + "tokens_seen": 315817984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045671013039117354, + "loss": 2.9749, + "theoretical_loss": 4.121500177266178, + "tokens_seen": 315883520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004567001003009027, + "loss": 2.9981, + "theoretical_loss": 4.121400703059497, + "tokens_seen": 315949056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566900702106319, + "loss": 3.0599, + "theoretical_loss": 4.121301255260217, + "tokens_seen": 316014592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566800401203611, + "loss": 2.9343, + "theoretical_loss": 4.121201833855856, + "tokens_seen": 316080128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566700100300903, + "loss": 3.0488, + "theoretical_loss": 4.121102438833935, + "tokens_seen": 316145664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 186091, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.028201103210449, + "objective/train/theoretical_loss": 4.121003070181987, + "objective/train/tokens_used": 336671200, + "theoretical_loss": 4.121003070181987, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045665997993981944, + "loss": 3.0888, + "theoretical_loss": 4.121003070181987, + "tokens_seen": 316211200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566499498495487, + "loss": 2.8606, + "theoretical_loss": 4.120903727887554, + "tokens_seen": 316276736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566399197592778, + "loss": 2.8707, + "theoretical_loss": 4.120804411938185, + "tokens_seen": 316342272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045662988966900704, + "loss": 3.1801, + "theoretical_loss": 4.120705122321437, + "tokens_seen": 316407808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566198595787362, + "loss": 2.9392, + "theoretical_loss": 4.1206058590248755, + "tokens_seen": 316473344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004566098294884654, + "loss": 3.0525, + "theoretical_loss": 4.120506622036077, + "tokens_seen": 316538880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565997993981946, + "loss": 2.8932, + "theoretical_loss": 4.120407411342623, + "tokens_seen": 316604416 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045658976930792376, + "loss": 3.0008, + "theoretical_loss": 4.1203082269321065, + "tokens_seen": 316669952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045657973921765294, + "loss": 2.9041, + "theoretical_loss": 4.120209068792127, + "tokens_seen": 316735488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565697091273822, + "loss": 2.8144, + "theoretical_loss": 4.120109936910291, + "tokens_seen": 316801024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565596790371113, + "loss": 2.9752, + "theoretical_loss": 4.120010831274218, + "tokens_seen": 316866560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045654964894684054, + "loss": 2.9861, + "theoretical_loss": 4.119911751871531, + "tokens_seen": 316932096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045653961885656967, + "loss": 3.1201, + "theoretical_loss": 4.1198126986898655, + "tokens_seen": 316997632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565295887662989, + "loss": 3.2564, + "theoretical_loss": 4.119713671716862, + "tokens_seen": 317063168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004565195586760281, + "loss": 3.0303, + "theoretical_loss": 4.11961467094017, + "tokens_seen": 317128704 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045650952858575727, + "loss": 3.0001, + "theoretical_loss": 4.119515696347451, + "tokens_seen": 317194240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045649949849548645, + "loss": 3.1713, + "theoretical_loss": 4.119416747926369, + "tokens_seen": 317259776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564894684052157, + "loss": 2.9097, + "theoretical_loss": 4.119317825664601, + "tokens_seen": 317325312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564794383149448, + "loss": 3.0256, + "theoretical_loss": 4.119218929549831, + "tokens_seen": 317390848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045646940822467405, + "loss": 2.9957, + "theoretical_loss": 4.119120059569751, + "tokens_seen": 317456384 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045645937813440317, + "loss": 3.1022, + "theoretical_loss": 4.11902121571206, + "tokens_seen": 317521920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004564493480441324, + "loss": 2.8814, + "theoretical_loss": 4.118922397964468, + "tokens_seen": 317587456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045643931795386164, + "loss": 2.8242, + "theoretical_loss": 4.118823606314692, + "tokens_seen": 317652992 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045642928786359077, + "loss": 3.1631, + "theoretical_loss": 4.118724840750456, + "tokens_seen": 317718528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045641925777332, + "loss": 3.0526, + "theoretical_loss": 4.118626101259496, + "tokens_seen": 317784064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 187059, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0989761352539062, + "objective/train/theoretical_loss": 4.118527387829552, + "objective/train/tokens_used": 338309600, + "theoretical_loss": 4.118527387829552, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045640922768304913, + "loss": 3.1599, + "theoretical_loss": 4.118527387829552, + "tokens_seen": 317849600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045639919759277837, + "loss": 2.9083, + "theoretical_loss": 4.118428700448375, + "tokens_seen": 317915136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045638916750250755, + "loss": 2.7942, + "theoretical_loss": 4.118330039103723, + "tokens_seen": 317980672 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045637913741223673, + "loss": 2.8139, + "theoretical_loss": 4.118231403783365, + "tokens_seen": 318046208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563691073219659, + "loss": 3.037, + "theoretical_loss": 4.118132794475073, + "tokens_seen": 318111744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045635907723169515, + "loss": 3.0491, + "theoretical_loss": 4.118034211166632, + "tokens_seen": 318177280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045634904714142427, + "loss": 2.8304, + "theoretical_loss": 4.117935653845834, + "tokens_seen": 318242816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004563390170511535, + "loss": 2.9257, + "theoretical_loss": 4.117837122500477, + "tokens_seen": 318308352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045632898696088263, + "loss": 3.1308, + "theoretical_loss": 4.117738617118372, + "tokens_seen": 318373888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045631895687061187, + "loss": 2.8832, + "theoretical_loss": 4.1176401376873315, + "tokens_seen": 318439424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045630892678034105, + "loss": 3.0881, + "theoretical_loss": 4.117541684195183, + "tokens_seen": 318504960 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045629889669007023, + "loss": 3.0655, + "theoretical_loss": 4.117443256629758, + "tokens_seen": 318570496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562888665997994, + "loss": 3.2203, + "theoretical_loss": 4.1173448549788985, + "tokens_seen": 318636032 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562788365095286, + "loss": 2.8932, + "theoretical_loss": 4.117246479230452, + "tokens_seen": 318701568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562688064192578, + "loss": 3.1329, + "theoretical_loss": 4.117148129372277, + "tokens_seen": 318767104 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456258776328987, + "loss": 3.0318, + "theoretical_loss": 4.117049805392239, + "tokens_seen": 318832640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045624874623871614, + "loss": 3.0924, + "theoretical_loss": 4.116951507278211, + "tokens_seen": 318898176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562387161484454, + "loss": 3.2415, + "theoretical_loss": 4.116853235018075, + "tokens_seen": 318963712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562286860581745, + "loss": 2.8784, + "theoretical_loss": 4.116754988599722, + "tokens_seen": 319029248 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045621865596790374, + "loss": 2.9519, + "theoretical_loss": 4.11665676801105, + "tokens_seen": 319094784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004562086258776329, + "loss": 3.0788, + "theoretical_loss": 4.116558573239965, + "tokens_seen": 319160320 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561985957873621, + "loss": 2.9773, + "theoretical_loss": 4.116460404274381, + "tokens_seen": 319225856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561885656970913, + "loss": 2.8958, + "theoretical_loss": 4.116362261102221, + "tokens_seen": 319291392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561785356068205, + "loss": 3.1937, + "theoretical_loss": 4.116264143711416, + "tokens_seen": 319356928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045616850551654964, + "loss": 3.0717, + "theoretical_loss": 4.1161660520899055, + "tokens_seen": 319422464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 187800, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.174799680709839, + "objective/train/theoretical_loss": 4.116067986225635, + "objective/train/tokens_used": 339948000, + "theoretical_loss": 4.116067986225635, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561584754262789, + "loss": 3.0015, + "theoretical_loss": 4.116067986225635, + "tokens_seen": 319488000 + }, + { + "epoch": 0.1, + "learning_rate": 0.000456148445336008, + "loss": 2.8217, + "theoretical_loss": 4.11596994610656, + "tokens_seen": 319553536 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045613841524573724, + "loss": 3.3545, + "theoretical_loss": 4.115871931720645, + "tokens_seen": 319619072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561283851554664, + "loss": 2.7344, + "theoretical_loss": 4.11577394305586, + "tokens_seen": 319684608 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561183550651956, + "loss": 2.8487, + "theoretical_loss": 4.115675980100185, + "tokens_seen": 319750144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004561083249749248, + "loss": 2.9135, + "theoretical_loss": 4.115578042841607, + "tokens_seen": 319815680 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045609829488465396, + "loss": 2.9709, + "theoretical_loss": 4.115480131268122, + "tokens_seen": 319881216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045608826479438314, + "loss": 2.9644, + "theoretical_loss": 4.1153822453677344, + "tokens_seen": 319946752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560782347041124, + "loss": 2.9821, + "theoretical_loss": 4.115284385128454, + "tokens_seen": 320012288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560682046138415, + "loss": 3.1831, + "theoretical_loss": 4.115186550538302, + "tokens_seen": 320077824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045605817452357074, + "loss": 3.0413, + "theoretical_loss": 4.115088741585305, + "tokens_seen": 320143360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045604814443329987, + "loss": 2.9108, + "theoretical_loss": 4.114990958257501, + "tokens_seen": 320208896 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560381143430291, + "loss": 3.2611, + "theoretical_loss": 4.11489320054293, + "tokens_seen": 320274432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004560280842527583, + "loss": 2.6443, + "theoretical_loss": 4.114795468429648, + "tokens_seen": 320339968 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045601805416248747, + "loss": 3.2413, + "theoretical_loss": 4.114697761905712, + "tokens_seen": 320405504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045600802407221665, + "loss": 3.2062, + "theoretical_loss": 4.1146000809591925, + "tokens_seen": 320471040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559979939819459, + "loss": 3.2569, + "theoretical_loss": 4.114502425578163, + "tokens_seen": 320536576 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455987963891675, + "loss": 3.047, + "theoretical_loss": 4.1144047957507075, + "tokens_seen": 320602112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045597793380140425, + "loss": 2.9388, + "theoretical_loss": 4.114307191464919, + "tokens_seen": 320667648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045596790371113337, + "loss": 3.0027, + "theoretical_loss": 4.114209612708898, + "tokens_seen": 320733184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559578736208626, + "loss": 2.8668, + "theoretical_loss": 4.1141120594707505, + "tokens_seen": 320798720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559478435305918, + "loss": 3.1505, + "theoretical_loss": 4.114014531738593, + "tokens_seen": 320864256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045593781344032097, + "loss": 3.0058, + "theoretical_loss": 4.113917029500549, + "tokens_seen": 320929792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045592778335005015, + "loss": 3.057, + "theoretical_loss": 4.113819552744752, + "tokens_seen": 320995328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045591775325977933, + "loss": 2.9634, + "theoretical_loss": 4.11372210145934, + "tokens_seen": 321060864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 188718, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.97586989402771, + "objective/train/theoretical_loss": 4.11362467563246, + "objective/train/tokens_used": 341586400, + "theoretical_loss": 4.11362467563246, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004559077231695085, + "loss": 3.0639, + "theoretical_loss": 4.11362467563246, + "tokens_seen": 321126400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045589769307923775, + "loss": 3.2388, + "theoretical_loss": 4.1135272752522685, + "tokens_seen": 321191936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558876629889669, + "loss": 3.1461, + "theoretical_loss": 4.1134299003069295, + "tokens_seen": 321257472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558776328986961, + "loss": 3.0283, + "theoretical_loss": 4.113332550784613, + "tokens_seen": 321323008 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045586760280842524, + "loss": 2.9909, + "theoretical_loss": 4.113235226673499, + "tokens_seen": 321388544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558575727181545, + "loss": 3.0179, + "theoretical_loss": 4.113137927961776, + "tokens_seen": 321454080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045584754262788365, + "loss": 2.8515, + "theoretical_loss": 4.113040654637637, + "tokens_seen": 321519616 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045583751253761284, + "loss": 2.755, + "theoretical_loss": 4.112943406689286, + "tokens_seen": 321585152 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455827482447342, + "loss": 2.9629, + "theoretical_loss": 4.112846184104933, + "tokens_seen": 321650688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045581745235707125, + "loss": 2.9968, + "theoretical_loss": 4.112748986872798, + "tokens_seen": 321716224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004558074222668004, + "loss": 3.2506, + "theoretical_loss": 4.112651814981108, + "tokens_seen": 321781760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557973921765296, + "loss": 3.103, + "theoretical_loss": 4.112554668418095, + "tokens_seen": 321847296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045578736208625874, + "loss": 3.1271, + "theoretical_loss": 4.112457547172005, + "tokens_seen": 321912832 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455777331995988, + "loss": 3.2188, + "theoretical_loss": 4.112360451231083, + "tokens_seen": 321978368 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045576730190571716, + "loss": 3.0657, + "theoretical_loss": 4.112263380583593, + "tokens_seen": 322043904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045575727181544634, + "loss": 3.1376, + "theoretical_loss": 4.112166335217797, + "tokens_seen": 322109440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557472417251755, + "loss": 2.8802, + "theoretical_loss": 4.112069315121969, + "tokens_seen": 322174976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557372116349047, + "loss": 3.0532, + "theoretical_loss": 4.111972320284392, + "tokens_seen": 322240512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557271815446339, + "loss": 2.9202, + "theoretical_loss": 4.111875350693354, + "tokens_seen": 322306048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004557171514543631, + "loss": 3.1109, + "theoretical_loss": 4.111778406337153, + "tokens_seen": 322371584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045570712136409224, + "loss": 2.8451, + "theoretical_loss": 4.111681487204094, + "tokens_seen": 322437120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556970912738215, + "loss": 2.957, + "theoretical_loss": 4.111584593282489, + "tokens_seen": 322502656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556870611835507, + "loss": 3.2971, + "theoretical_loss": 4.111487724560659, + "tokens_seen": 322568192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045567703109327984, + "loss": 3.227, + "theoretical_loss": 4.111390881026932, + "tokens_seen": 322633728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556670010030091, + "loss": 3.1005, + "theoretical_loss": 4.111294062669645, + "tokens_seen": 322699264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 189140, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8794915676116943, + "objective/train/theoretical_loss": 4.111197269477141, + "objective/train/tokens_used": 343224800, + "theoretical_loss": 4.111197269477141, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556569709127382, + "loss": 2.9512, + "theoretical_loss": 4.111197269477141, + "tokens_seen": 322764800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045564694082246744, + "loss": 3.0384, + "theoretical_loss": 4.111100501437772, + "tokens_seen": 322830336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556369107321966, + "loss": 2.8792, + "theoretical_loss": 4.111003758539896, + "tokens_seen": 322895872 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004556268806419258, + "loss": 3.1337, + "theoretical_loss": 4.110907040771883, + "tokens_seen": 322961408 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455616850551655, + "loss": 3.186, + "theoretical_loss": 4.110810348122105, + "tokens_seen": 323026944 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045560682046138416, + "loss": 2.9821, + "theoretical_loss": 4.110713680578947, + "tokens_seen": 323092480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045559679037111334, + "loss": 3.2378, + "theoretical_loss": 4.110617038130797, + "tokens_seen": 323158016 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555867602808426, + "loss": 2.6982, + "theoretical_loss": 4.110520420766054, + "tokens_seen": 323223552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555767301905717, + "loss": 3.3153, + "theoretical_loss": 4.110423828473126, + "tokens_seen": 323289088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045556670010030094, + "loss": 3.0101, + "theoretical_loss": 4.110327261240422, + "tokens_seen": 323354624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045555667001003007, + "loss": 3.1061, + "theoretical_loss": 4.110230719056367, + "tokens_seen": 323420160 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555466399197593, + "loss": 3.1276, + "theoretical_loss": 4.110134201909388, + "tokens_seen": 323485696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555366098294885, + "loss": 3.2466, + "theoretical_loss": 4.110037709787923, + "tokens_seen": 323551232 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045552657973921767, + "loss": 3.347, + "theoretical_loss": 4.109941242680414, + "tokens_seen": 323616768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045551654964894685, + "loss": 3.1481, + "theoretical_loss": 4.109844800575315, + "tokens_seen": 323682304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004555065195586761, + "loss": 2.9884, + "theoretical_loss": 4.109748383461085, + "tokens_seen": 323747840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554964894684052, + "loss": 3.1188, + "theoretical_loss": 4.10965199132619, + "tokens_seen": 323813376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045548645937813445, + "loss": 3.2839, + "theoretical_loss": 4.109555624159107, + "tokens_seen": 323878912 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045547642928786357, + "loss": 3.3218, + "theoretical_loss": 4.109459281948317, + "tokens_seen": 323944448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554663991975928, + "loss": 2.8989, + "theoretical_loss": 4.109362964682312, + "tokens_seen": 324009984 + }, + { + "epoch": 0.1, + "learning_rate": 0.000455456369107322, + "loss": 2.9077, + "theoretical_loss": 4.1092666723495865, + "tokens_seen": 324075520 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045544633901705117, + "loss": 3.2878, + "theoretical_loss": 4.109170404938649, + "tokens_seen": 324141056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045543630892678035, + "loss": 3.2166, + "theoretical_loss": 4.109074162438011, + "tokens_seen": 324206592 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045542627883650953, + "loss": 2.8978, + "theoretical_loss": 4.1089779448361945, + "tokens_seen": 324272128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004554162487462387, + "loss": 3.1213, + "theoretical_loss": 4.108881752121727, + "tokens_seen": 324337664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 190150, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1001152992248535, + "objective/train/theoretical_loss": 4.108785584283144, + "objective/train/tokens_used": 344863200, + "theoretical_loss": 4.108785584283144, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045540621865596795, + "loss": 2.9022, + "theoretical_loss": 4.108785584283144, + "tokens_seen": 324403200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553961885656971, + "loss": 2.9865, + "theoretical_loss": 4.108689441308989, + "tokens_seen": 324468736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553861584754263, + "loss": 3.112, + "theoretical_loss": 4.108593323187815, + "tokens_seen": 324534272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045537612838515544, + "loss": 3.1477, + "theoretical_loss": 4.108497229908179, + "tokens_seen": 324599808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553660982948847, + "loss": 3.0519, + "theoretical_loss": 4.108401161458646, + "tokens_seen": 324665344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045535606820461385, + "loss": 3.0665, + "theoretical_loss": 4.1083051178277925, + "tokens_seen": 324730880 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045534603811434304, + "loss": 2.968, + "theoretical_loss": 4.108209099004198, + "tokens_seen": 324796416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553360080240722, + "loss": 3.1983, + "theoretical_loss": 4.108113104976453, + "tokens_seen": 324861952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045532597793380145, + "loss": 2.8336, + "theoretical_loss": 4.108017135733152, + "tokens_seen": 324927488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553159478435306, + "loss": 3.0941, + "theoretical_loss": 4.107921191262901, + "tokens_seen": 324993024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004553059177532598, + "loss": 2.9691, + "theoretical_loss": 4.10782527155431, + "tokens_seen": 325058560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045529588766298894, + "loss": 2.9584, + "theoretical_loss": 4.107729376595999, + "tokens_seen": 325124096 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552858575727182, + "loss": 3.253, + "theoretical_loss": 4.107633506376594, + "tokens_seen": 325189632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045527582748244736, + "loss": 3.0396, + "theoretical_loss": 4.10753766088473, + "tokens_seen": 325255168 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045526579739217654, + "loss": 3.0754, + "theoretical_loss": 4.107441840109048, + "tokens_seen": 325320704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552557673019057, + "loss": 3.0014, + "theoretical_loss": 4.107346044038197, + "tokens_seen": 325386240 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552457372116349, + "loss": 3.0876, + "theoretical_loss": 4.107250272660834, + "tokens_seen": 325451776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552357071213641, + "loss": 2.9613, + "theoretical_loss": 4.107154525965623, + "tokens_seen": 325517312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552256770310933, + "loss": 2.9588, + "theoretical_loss": 4.107058803941236, + "tokens_seen": 325582848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045521564694082244, + "loss": 2.9034, + "theoretical_loss": 4.106963106576352, + "tokens_seen": 325648384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004552056168505517, + "loss": 2.8674, + "theoretical_loss": 4.106867433859656, + "tokens_seen": 325713920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551955867602808, + "loss": 2.9357, + "theoretical_loss": 4.106771785779845, + "tokens_seen": 325779456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045518555667001004, + "loss": 2.9561, + "theoretical_loss": 4.106676162325618, + "tokens_seen": 325844992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551755265797392, + "loss": 3.0155, + "theoretical_loss": 4.106580563485685, + "tokens_seen": 325910528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551654964894684, + "loss": 3.0204, + "theoretical_loss": 4.106484989248762, + "tokens_seen": 325976064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 191328, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0341267585754395, + "objective/train/theoretical_loss": 4.106389439603573, + "objective/train/tokens_used": 346501600, + "theoretical_loss": 4.106389439603573, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551554663991976, + "loss": 3.0402, + "theoretical_loss": 4.106389439603573, + "tokens_seen": 326041600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551454363089268, + "loss": 3.1653, + "theoretical_loss": 4.10629391453885, + "tokens_seen": 326107136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045513540621865595, + "loss": 2.9886, + "theoretical_loss": 4.10619841404333, + "tokens_seen": 326172672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551253761283852, + "loss": 3.3446, + "theoretical_loss": 4.106102938105762, + "tokens_seen": 326238208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004551153460381143, + "loss": 2.8962, + "theoretical_loss": 4.106007486714896, + "tokens_seen": 326303744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045510531594784354, + "loss": 3.1154, + "theoretical_loss": 4.105912059859495, + "tokens_seen": 326369280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550952858575727, + "loss": 3.048, + "theoretical_loss": 4.105816657528327, + "tokens_seen": 326434816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550852557673019, + "loss": 2.9685, + "theoretical_loss": 4.105721279710168, + "tokens_seen": 326500352 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550752256770311, + "loss": 3.1754, + "theoretical_loss": 4.105625926393801, + "tokens_seen": 326565888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045506519558676027, + "loss": 3.2148, + "theoretical_loss": 4.105530597568015, + "tokens_seen": 326631424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045505516549648945, + "loss": 3.1651, + "theoretical_loss": 4.1054352932216105, + "tokens_seen": 326696960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550451354062187, + "loss": 3.0107, + "theoretical_loss": 4.10534001334339, + "tokens_seen": 326762496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550351053159478, + "loss": 3.2755, + "theoretical_loss": 4.105244757922169, + "tokens_seen": 326828032 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045502507522567705, + "loss": 2.8913, + "theoretical_loss": 4.105149526946766, + "tokens_seen": 326893568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045501504513540623, + "loss": 2.9583, + "theoretical_loss": 4.1050543204060075, + "tokens_seen": 326959104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004550050150451354, + "loss": 3.187, + "theoretical_loss": 4.10495913828873, + "tokens_seen": 327024640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549949849548646, + "loss": 2.9074, + "theoretical_loss": 4.104863980583774, + "tokens_seen": 327090176 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045498495486459377, + "loss": 3.144, + "theoretical_loss": 4.104768847279988, + "tokens_seen": 327155712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045497492477432295, + "loss": 3.0299, + "theoretical_loss": 4.104673738366232, + "tokens_seen": 327221248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549648946840522, + "loss": 2.9183, + "theoretical_loss": 4.104578653831368, + "tokens_seen": 327286784 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045495486459378137, + "loss": 2.8461, + "theoretical_loss": 4.104483593664266, + "tokens_seen": 327352320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045494483450351055, + "loss": 2.9215, + "theoretical_loss": 4.104388557853807, + "tokens_seen": 327417856 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045493480441323973, + "loss": 3.0317, + "theoretical_loss": 4.104293546388876, + "tokens_seen": 327483392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549247743229689, + "loss": 3.0664, + "theoretical_loss": 4.104198559258364, + "tokens_seen": 327548928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045491474423269815, + "loss": 2.9383, + "theoretical_loss": 4.104103596451175, + "tokens_seen": 327614464 + }, + { + "debugging/Self-BLEU-5": 0.2643082570774211, + "debugging/distinct-1-grams": 0.8218199262737969, + "debugging/distinct-2-grams": 0.9963369963369964, + "debugging/entropy-1-grams": 4.917654182369999, + "debugging/entropy-2-grams": 5.417185152102512, + "debugging/length": 514.3333333333334, + "debugging/num_segments": 3, + "epoch": 0.1, + "objective/train/docs_used": 191945, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.493633270263672, + "objective/train/theoretical_loss": 4.104008657956216, + "objective/train/tokens_used": 348140000, + "theoretical_loss": 4.104008657956216, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004549047141424273, + "loss": 3.1165, + "theoretical_loss": 4.104008657956216, + "tokens_seen": 327680000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548946840521565, + "loss": 2.9939, + "theoretical_loss": 4.1039137437624, + "tokens_seen": 327745536 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045488465396188564, + "loss": 2.8305, + "theoretical_loss": 4.103818853858651, + "tokens_seen": 327811072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548746238716149, + "loss": 3.0165, + "theoretical_loss": 4.103723988233898, + "tokens_seen": 327876608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045486459378134405, + "loss": 3.1865, + "theoretical_loss": 4.103629146877078, + "tokens_seen": 327942144 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045485456369107324, + "loss": 3.0442, + "theoretical_loss": 4.103534329777135, + "tokens_seen": 328007680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548445336008024, + "loss": 2.9175, + "theoretical_loss": 4.103439536923021, + "tokens_seen": 328073216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045483450351053165, + "loss": 2.9148, + "theoretical_loss": 4.103344768303694, + "tokens_seen": 328138752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004548244734202608, + "loss": 3.2821, + "theoretical_loss": 4.103250023908119, + "tokens_seen": 328204288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045481444332999, + "loss": 3.0728, + "theoretical_loss": 4.103155303725271, + "tokens_seen": 328269824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045480441323971914, + "loss": 2.9522, + "theoretical_loss": 4.103060607744128, + "tokens_seen": 328335360 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547943831494484, + "loss": 3.0337, + "theoretical_loss": 4.102965935953679, + "tokens_seen": 328400896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045478435305917756, + "loss": 2.7991, + "theoretical_loss": 4.102871288342918, + "tokens_seen": 328466432 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045477432296890674, + "loss": 2.9165, + "theoretical_loss": 4.102776664900848, + "tokens_seen": 328531968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547642928786359, + "loss": 3.0264, + "theoretical_loss": 4.1026820656164755, + "tokens_seen": 328597504 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547542627883651, + "loss": 2.956, + "theoretical_loss": 4.102587490478819, + "tokens_seen": 328663040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547442326980943, + "loss": 3.0012, + "theoretical_loss": 4.102492939476901, + "tokens_seen": 328728576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547342026078235, + "loss": 3.2309, + "theoretical_loss": 4.102398412599753, + "tokens_seen": 328794112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045472417251755264, + "loss": 2.9335, + "theoretical_loss": 4.102303909836412, + "tokens_seen": 328859648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004547141424272819, + "loss": 3.0824, + "theoretical_loss": 4.102209431175923, + "tokens_seen": 328925184 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454704112337011, + "loss": 3.0037, + "theoretical_loss": 4.102114976607338, + "tokens_seen": 328990720 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045469408224674024, + "loss": 2.9785, + "theoretical_loss": 4.102020546119716, + "tokens_seen": 329056256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546840521564694, + "loss": 3.0101, + "theoretical_loss": 4.101926139702125, + "tokens_seen": 329121792 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546740220661986, + "loss": 3.1876, + "theoretical_loss": 4.101831757343637, + "tokens_seen": 329187328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546639919759278, + "loss": 3.0554, + "theoretical_loss": 4.1017373990333335, + "tokens_seen": 329252864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 193205, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.53017520904541, + "objective/train/theoretical_loss": 4.101643064760301, + "objective/train/tokens_used": 349778400, + "theoretical_loss": 4.101643064760301, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454653961885657, + "loss": 2.8743, + "theoretical_loss": 4.101643064760301, + "tokens_seen": 329318400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045464393179538615, + "loss": 2.7826, + "theoretical_loss": 4.101548754513636, + "tokens_seen": 329383936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546339017051154, + "loss": 2.8613, + "theoretical_loss": 4.101454468282439, + "tokens_seen": 329449472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546238716148445, + "loss": 2.9591, + "theoretical_loss": 4.101360206055821, + "tokens_seen": 329515008 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045461384152457374, + "loss": 3.0603, + "theoretical_loss": 4.101265967822897, + "tokens_seen": 329580544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004546038114343029, + "loss": 2.8043, + "theoretical_loss": 4.101171753572791, + "tokens_seen": 329646080 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545937813440321, + "loss": 3.0562, + "theoretical_loss": 4.101077563294632, + "tokens_seen": 329711616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545837512537613, + "loss": 3.1101, + "theoretical_loss": 4.1009833969775595, + "tokens_seen": 329777152 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045457372116349047, + "loss": 3.162, + "theoretical_loss": 4.100889254610717, + "tokens_seen": 329842688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045456369107321965, + "loss": 2.9801, + "theoretical_loss": 4.100795136183256, + "tokens_seen": 329908224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545536609829489, + "loss": 2.9315, + "theoretical_loss": 4.100701041684336, + "tokens_seen": 329973760 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454543630892678, + "loss": 3.097, + "theoretical_loss": 4.100606971103122, + "tokens_seen": 330039296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045453360080240725, + "loss": 3.0641, + "theoretical_loss": 4.100512924428788, + "tokens_seen": 330104832 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045452357071213643, + "loss": 2.9796, + "theoretical_loss": 4.100418901650513, + "tokens_seen": 330170368 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545135406218656, + "loss": 2.9823, + "theoretical_loss": 4.1003249027574835, + "tokens_seen": 330235904 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004545035105315948, + "loss": 3.0358, + "theoretical_loss": 4.100230927738895, + "tokens_seen": 330301440 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045449348044132397, + "loss": 3.0782, + "theoretical_loss": 4.100136976583947, + "tokens_seen": 330366976 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045448345035105315, + "loss": 3.0677, + "theoretical_loss": 4.100043049281849, + "tokens_seen": 330432512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544734202607824, + "loss": 2.9041, + "theoretical_loss": 4.099949145821815, + "tokens_seen": 330498048 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544633901705115, + "loss": 3.1047, + "theoretical_loss": 4.099855266193067, + "tokens_seen": 330563584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045445336008024075, + "loss": 3.2029, + "theoretical_loss": 4.099761410384835, + "tokens_seen": 330629120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544433299899699, + "loss": 2.8177, + "theoretical_loss": 4.099667578386355, + "tokens_seen": 330694656 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544332998996991, + "loss": 2.8579, + "theoretical_loss": 4.099573770186869, + "tokens_seen": 330760192 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544232698094283, + "loss": 2.9416, + "theoretical_loss": 4.099479985775629, + "tokens_seen": 330825728 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004544132397191575, + "loss": 2.997, + "theoretical_loss": 4.09938622514189, + "tokens_seen": 330891264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 193823, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6776926517486572, + "objective/train/theoretical_loss": 4.099292488274917, + "objective/train/tokens_used": 351416800, + "theoretical_loss": 4.099292488274917, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045440320962888666, + "loss": 2.9025, + "theoretical_loss": 4.099292488274917, + "tokens_seen": 330956800 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045439317953861584, + "loss": 2.9272, + "theoretical_loss": 4.0991987751639805, + "tokens_seen": 331022336 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454383149448345, + "loss": 3.0293, + "theoretical_loss": 4.099105085798359, + "tokens_seen": 331087872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045437311935807425, + "loss": 2.9882, + "theoretical_loss": 4.099011420167338, + "tokens_seen": 331153408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543630892678034, + "loss": 2.9665, + "theoretical_loss": 4.098917778260208, + "tokens_seen": 331218944 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543530591775326, + "loss": 3.145, + "theoretical_loss": 4.098824160066269, + "tokens_seen": 331284480 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543430290872618, + "loss": 3.0804, + "theoretical_loss": 4.098730565574826, + "tokens_seen": 331350016 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454332998996991, + "loss": 2.8699, + "theoretical_loss": 4.098636994775193, + "tokens_seen": 331415552 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045432296890672016, + "loss": 2.8012, + "theoretical_loss": 4.098543447656688, + "tokens_seen": 331481088 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045431293881644934, + "loss": 3.1938, + "theoretical_loss": 4.098449924208639, + "tokens_seen": 331546624 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004543029087261785, + "loss": 2.8317, + "theoretical_loss": 4.098356424420379, + "tokens_seen": 331612160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045429287863590776, + "loss": 2.9114, + "theoretical_loss": 4.098262948281247, + "tokens_seen": 331677696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542828485456369, + "loss": 2.9306, + "theoretical_loss": 4.098169495780592, + "tokens_seen": 331743232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542728184553661, + "loss": 3.0543, + "theoretical_loss": 4.098076066907769, + "tokens_seen": 331808768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045426278836509525, + "loss": 2.9948, + "theoretical_loss": 4.0979826616521375, + "tokens_seen": 331874304 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542527582748245, + "loss": 3.0341, + "theoretical_loss": 4.0978892800030655, + "tokens_seen": 331939840 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045424272818455366, + "loss": 2.7333, + "theoretical_loss": 4.0977959219499285, + "tokens_seen": 332005376 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045423269809428284, + "loss": 2.7174, + "theoretical_loss": 4.097702587482107, + "tokens_seen": 332070912 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454222668004012, + "loss": 2.9369, + "theoretical_loss": 4.097609276588993, + "tokens_seen": 332136448 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004542126379137412, + "loss": 2.9834, + "theoretical_loss": 4.097515989259978, + "tokens_seen": 332201984 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045420260782347044, + "loss": 2.7845, + "theoretical_loss": 4.097422725484467, + "tokens_seen": 332267520 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541925777331996, + "loss": 2.7439, + "theoretical_loss": 4.097329485251869, + "tokens_seen": 332333056 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541825476429288, + "loss": 3.1478, + "theoretical_loss": 4.097236268551598, + "tokens_seen": 332398592 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454172517552658, + "loss": 2.7357, + "theoretical_loss": 4.097143075373079, + "tokens_seen": 332464128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541624874623872, + "loss": 2.9623, + "theoretical_loss": 4.097049905705742, + "tokens_seen": 332529664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 194956, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.295325756072998, + "objective/train/theoretical_loss": 4.0969567595390215, + "objective/train/tokens_used": 353055200, + "theoretical_loss": 4.0969567595390215, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045415245737211635, + "loss": 3.0697, + "theoretical_loss": 4.0969567595390215, + "tokens_seen": 332595200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541424272818456, + "loss": 2.9866, + "theoretical_loss": 4.096863636862363, + "tokens_seen": 332660736 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541323971915747, + "loss": 3.0612, + "theoretical_loss": 4.096770537665216, + "tokens_seen": 332726272 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045412236710130395, + "loss": 2.8517, + "theoretical_loss": 4.096677461937037, + "tokens_seen": 332791808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541123370110331, + "loss": 3.2886, + "theoretical_loss": 4.09658440966729, + "tokens_seen": 332857344 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004541023069207623, + "loss": 3.0414, + "theoretical_loss": 4.096491380845446, + "tokens_seen": 332922880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540922768304915, + "loss": 3.1045, + "theoretical_loss": 4.096398375460983, + "tokens_seen": 332988416 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045408224674022067, + "loss": 3.19, + "theoretical_loss": 4.096305393503385, + "tokens_seen": 333053952 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045407221664994985, + "loss": 2.9571, + "theoretical_loss": 4.096212434962142, + "tokens_seen": 333119488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540621865596791, + "loss": 2.8609, + "theoretical_loss": 4.096119499826752, + "tokens_seen": 333185024 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540521564694082, + "loss": 3.079, + "theoretical_loss": 4.096026588086722, + "tokens_seen": 333250560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045404212637913745, + "loss": 2.9375, + "theoretical_loss": 4.09593369973156, + "tokens_seen": 333316096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045403209628886663, + "loss": 3.0762, + "theoretical_loss": 4.095840834750787, + "tokens_seen": 333381632 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004540220661985958, + "loss": 3.0485, + "theoretical_loss": 4.095747993133925, + "tokens_seen": 333447168 + }, + { + "epoch": 0.1, + "learning_rate": 0.000454012036108325, + "loss": 2.8056, + "theoretical_loss": 4.095655174870508, + "tokens_seen": 333512704 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045400200601805417, + "loss": 2.9316, + "theoretical_loss": 4.095562379950073, + "tokens_seen": 333578240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045399197592778335, + "loss": 3.156, + "theoretical_loss": 4.0954696083621664, + "tokens_seen": 333643776 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539819458375126, + "loss": 2.8985, + "theoretical_loss": 4.095376860096339, + "tokens_seen": 333709312 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539719157472417, + "loss": 2.944, + "theoretical_loss": 4.095284135142149, + "tokens_seen": 333774848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045396188565697095, + "loss": 2.8726, + "theoretical_loss": 4.095191433489164, + "tokens_seen": 333840384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539518555667001, + "loss": 2.8167, + "theoretical_loss": 4.095098755126954, + "tokens_seen": 333905920 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539418254764293, + "loss": 2.859, + "theoretical_loss": 4.095006100045097, + "tokens_seen": 333971456 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539317953861585, + "loss": 2.9604, + "theoretical_loss": 4.094913468233182, + "tokens_seen": 334036992 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004539217652958877, + "loss": 2.9834, + "theoretical_loss": 4.094820859680797, + "tokens_seen": 334102528 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045391173520561686, + "loss": 2.9309, + "theoretical_loss": 4.094728274377543, + "tokens_seen": 334168064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 196277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.767504930496216, + "objective/train/theoretical_loss": 4.094635712313026, + "objective/train/tokens_used": 354693600, + "theoretical_loss": 4.094635712313026, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045390170511534604, + "loss": 3.0447, + "theoretical_loss": 4.094635712313026, + "tokens_seen": 334233600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538916750250752, + "loss": 3.0314, + "theoretical_loss": 4.094543173476857, + "tokens_seen": 334299136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045388164493480445, + "loss": 3.1375, + "theoretical_loss": 4.094450657858655, + "tokens_seen": 334364672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538716148445336, + "loss": 2.9004, + "theoretical_loss": 4.094358165448046, + "tokens_seen": 334430208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538615847542628, + "loss": 2.936, + "theoretical_loss": 4.094265696234663, + "tokens_seen": 334495744 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453851554663992, + "loss": 2.9983, + "theoretical_loss": 4.094173250208143, + "tokens_seen": 334561280 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538415245737212, + "loss": 3.0602, + "theoretical_loss": 4.094080827358133, + "tokens_seen": 334626816 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045383149448345036, + "loss": 2.8065, + "theoretical_loss": 4.093988427674285, + "tokens_seen": 334692352 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045382146439317954, + "loss": 2.8173, + "theoretical_loss": 4.093896051146257, + "tokens_seen": 334757888 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004538114343029087, + "loss": 2.9987, + "theoretical_loss": 4.0938036977637156, + "tokens_seen": 334823424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045380140421263796, + "loss": 2.9723, + "theoretical_loss": 4.093711367516333, + "tokens_seen": 334888960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537913741223671, + "loss": 3.1855, + "theoretical_loss": 4.093619060393786, + "tokens_seen": 334954496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537813440320963, + "loss": 3.0542, + "theoretical_loss": 4.0935267763857635, + "tokens_seen": 335020032 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045377131394182545, + "loss": 3.0719, + "theoretical_loss": 4.093434515481954, + "tokens_seen": 335085568 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537612838515547, + "loss": 2.7748, + "theoretical_loss": 4.093342277672058, + "tokens_seen": 335151104 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045375125376128386, + "loss": 2.9491, + "theoretical_loss": 4.09325006294578, + "tokens_seen": 335216640 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045374122367101304, + "loss": 2.9779, + "theoretical_loss": 4.093157871292833, + "tokens_seen": 335282176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537311935807422, + "loss": 2.8597, + "theoretical_loss": 4.093065702702933, + "tokens_seen": 335347712 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537211634904714, + "loss": 3.029, + "theoretical_loss": 4.0929735571658075, + "tokens_seen": 335413248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537111334002006, + "loss": 3.136, + "theoretical_loss": 4.092881434671186, + "tokens_seen": 335478784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004537011033099298, + "loss": 3.1326, + "theoretical_loss": 4.09278933520881, + "tokens_seen": 335544320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045369107321965895, + "loss": 2.7854, + "theoretical_loss": 4.092697258768419, + "tokens_seen": 335609856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536810431293882, + "loss": 3.006, + "theoretical_loss": 4.09260520533977, + "tokens_seen": 335675392 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045367101303911737, + "loss": 3.2209, + "theoretical_loss": 4.092513174912616, + "tokens_seen": 335740928 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045366098294884655, + "loss": 3.0421, + "theoretical_loss": 4.092421167476726, + "tokens_seen": 335806464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 196731, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1563823223114014, + "objective/train/theoretical_loss": 4.0923291830218655, + "objective/train/tokens_used": 356332000, + "theoretical_loss": 4.0923291830218655, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045365095285857573, + "loss": 3.0343, + "theoretical_loss": 4.0923291830218655, + "tokens_seen": 335872000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536409227683049, + "loss": 3.1099, + "theoretical_loss": 4.092237221537817, + "tokens_seen": 335937536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536308926780341, + "loss": 3.1611, + "theoretical_loss": 4.0921452830143625, + "tokens_seen": 336003072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536208625877633, + "loss": 3.0936, + "theoretical_loss": 4.092053367441292, + "tokens_seen": 336068608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045361083249749245, + "loss": 3.0695, + "theoretical_loss": 4.091961474808404, + "tokens_seen": 336134144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004536008024072217, + "loss": 3.1083, + "theoretical_loss": 4.091869605105501, + "tokens_seen": 336199680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535907723169508, + "loss": 2.9725, + "theoretical_loss": 4.091777758322394, + "tokens_seen": 336265216 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045358074222668005, + "loss": 2.7863, + "theoretical_loss": 4.091685934448899, + "tokens_seen": 336330752 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045357071213640923, + "loss": 2.7613, + "theoretical_loss": 4.09159413347484, + "tokens_seen": 336396288 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535606820461384, + "loss": 2.7013, + "theoretical_loss": 4.091502355390046, + "tokens_seen": 336461824 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535506519558676, + "loss": 3.0298, + "theoretical_loss": 4.091410600184353, + "tokens_seen": 336527360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045354062186559683, + "loss": 2.8801, + "theoretical_loss": 4.091318867847605, + "tokens_seen": 336592896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045353059177532596, + "loss": 2.9488, + "theoretical_loss": 4.091227158369649, + "tokens_seen": 336658432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535205616850552, + "loss": 2.8764, + "theoretical_loss": 4.091135471740342, + "tokens_seen": 336723968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004535105315947843, + "loss": 2.994, + "theoretical_loss": 4.091043807949546, + "tokens_seen": 336789504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045350050150451355, + "loss": 2.947, + "theoretical_loss": 4.0909521669871305, + "tokens_seen": 336855040 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045349047141424274, + "loss": 3.1228, + "theoretical_loss": 4.0908605488429695, + "tokens_seen": 336920576 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534804413239719, + "loss": 2.939, + "theoretical_loss": 4.090768953506944, + "tokens_seen": 336986112 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534704112337011, + "loss": 2.8694, + "theoretical_loss": 4.090677380968943, + "tokens_seen": 337051648 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534603811434303, + "loss": 2.9003, + "theoretical_loss": 4.09058583121886, + "tokens_seen": 337117184 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534503510531595, + "loss": 2.9993, + "theoretical_loss": 4.090494304246597, + "tokens_seen": 337182720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534403209628887, + "loss": 3.1168, + "theoretical_loss": 4.09040280004206, + "tokens_seen": 337248256 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534302908726179, + "loss": 2.9275, + "theoretical_loss": 4.090311318595164, + "tokens_seen": 337313792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045342026078234706, + "loss": 2.941, + "theoretical_loss": 4.090219859895829, + "tokens_seen": 337379328 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045341023069207624, + "loss": 2.8286, + "theoretical_loss": 4.09012842393398, + "tokens_seen": 337444864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 198039, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.190596342086792, + "objective/train/theoretical_loss": 4.090037010699552, + "objective/train/tokens_used": 357970400, + "theoretical_loss": 4.090037010699552, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004534002006018054, + "loss": 3.0192, + "theoretical_loss": 4.090037010699552, + "tokens_seen": 337510400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045339017051153465, + "loss": 2.921, + "theoretical_loss": 4.089945620182483, + "tokens_seen": 337575936 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533801404212638, + "loss": 3.1205, + "theoretical_loss": 4.08985425237272, + "tokens_seen": 337641472 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453370110330993, + "loss": 2.8657, + "theoretical_loss": 4.089762907260214, + "tokens_seen": 337707008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533600802407222, + "loss": 2.9334, + "theoretical_loss": 4.0896715848349245, + "tokens_seen": 337772544 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533500501504514, + "loss": 2.8687, + "theoretical_loss": 4.089580285086815, + "tokens_seen": 337838080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045334002006018056, + "loss": 2.9244, + "theoretical_loss": 4.0894890080058595, + "tokens_seen": 337903616 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045332998996990974, + "loss": 2.7939, + "theoretical_loss": 4.089397753582034, + "tokens_seen": 337969152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004533199598796389, + "loss": 2.9034, + "theoretical_loss": 4.089306521805322, + "tokens_seen": 338034688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045330992978936816, + "loss": 2.8053, + "theoretical_loss": 4.089215312665716, + "tokens_seen": 338100224 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532998996990973, + "loss": 3.0222, + "theoretical_loss": 4.0891241261532105, + "tokens_seen": 338165760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532898696088265, + "loss": 3.2622, + "theoretical_loss": 4.089032962257811, + "tokens_seen": 338231296 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045327983951855565, + "loss": 3.0372, + "theoretical_loss": 4.088941820969525, + "tokens_seen": 338296832 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532698094282849, + "loss": 2.8771, + "theoretical_loss": 4.08885070227837, + "tokens_seen": 338362368 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045325977933801406, + "loss": 2.9213, + "theoretical_loss": 4.088759606174369, + "tokens_seen": 338427904 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045324974924774324, + "loss": 2.877, + "theoretical_loss": 4.088668532647548, + "tokens_seen": 338493440 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532397191574724, + "loss": 2.9616, + "theoretical_loss": 4.088577481687944, + "tokens_seen": 338558976 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532296890672016, + "loss": 2.9701, + "theoretical_loss": 4.088486453285596, + "tokens_seen": 338624512 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004532196589769308, + "loss": 2.9765, + "theoretical_loss": 4.088395447430554, + "tokens_seen": 338690048 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045320962888666, + "loss": 2.9125, + "theoretical_loss": 4.088304464112872, + "tokens_seen": 338755584 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045319959879638915, + "loss": 2.7839, + "theoretical_loss": 4.088213503322607, + "tokens_seen": 338821120 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531895687061184, + "loss": 2.8615, + "theoretical_loss": 4.088122565049829, + "tokens_seen": 338886656 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045317953861584757, + "loss": 3.0402, + "theoretical_loss": 4.088031649284609, + "tokens_seen": 338952192 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045316950852557675, + "loss": 3.1794, + "theoretical_loss": 4.087940756017027, + "tokens_seen": 339017728 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045315947843530593, + "loss": 2.8781, + "theoretical_loss": 4.087849885237168, + "tokens_seen": 339083264 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 198530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.366729259490967, + "objective/train/theoretical_loss": 4.0877590369351235, + "objective/train/tokens_used": 359608800, + "theoretical_loss": 4.0877590369351235, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531494483450351, + "loss": 3.1109, + "theoretical_loss": 4.0877590369351235, + "tokens_seen": 339148800 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531394182547643, + "loss": 2.8525, + "theoretical_loss": 4.0876682111009925, + "tokens_seen": 339214336 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531293881644935, + "loss": 3.1438, + "theoretical_loss": 4.087577407724877, + "tokens_seen": 339279872 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045311935807422265, + "loss": 2.836, + "theoretical_loss": 4.08748662679689, + "tokens_seen": 339345408 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004531093279839519, + "loss": 2.8101, + "theoretical_loss": 4.087395868307146, + "tokens_seen": 339410944 + }, + { + "epoch": 0.1, + "learning_rate": 0.000453099297893681, + "loss": 3.1357, + "theoretical_loss": 4.08730513224577, + "tokens_seen": 339476480 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045308926780341025, + "loss": 3.14, + "theoretical_loss": 4.08721441860289, + "tokens_seen": 339542016 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045307923771313943, + "loss": 3.0339, + "theoretical_loss": 4.087123727368642, + "tokens_seen": 339607552 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530692076228686, + "loss": 3.1427, + "theoretical_loss": 4.087033058533167, + "tokens_seen": 339673088 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530591775325978, + "loss": 3.0171, + "theoretical_loss": 4.086942412086614, + "tokens_seen": 339738624 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045304914744232703, + "loss": 3.0101, + "theoretical_loss": 4.086851788019137, + "tokens_seen": 339804160 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045303911735205616, + "loss": 3.0004, + "theoretical_loss": 4.086761186320897, + "tokens_seen": 339869696 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530290872617854, + "loss": 3.0109, + "theoretical_loss": 4.086670606982059, + "tokens_seen": 339935232 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004530190571715145, + "loss": 2.7429, + "theoretical_loss": 4.086580049992798, + "tokens_seen": 340000768 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045300902708124375, + "loss": 2.9896, + "theoretical_loss": 4.086489515343292, + "tokens_seen": 340066304 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045299899699097294, + "loss": 3.0274, + "theoretical_loss": 4.086399003023725, + "tokens_seen": 340131840 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529889669007021, + "loss": 3.1022, + "theoretical_loss": 4.086308513024291, + "tokens_seen": 340197376 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529789368104313, + "loss": 2.9777, + "theoretical_loss": 4.086218045335187, + "tokens_seen": 340262912 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529689067201605, + "loss": 3.1257, + "theoretical_loss": 4.086127599946616, + "tokens_seen": 340328448 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045295887662988966, + "loss": 3.0384, + "theoretical_loss": 4.086037176848789, + "tokens_seen": 340393984 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529488465396189, + "loss": 3.0784, + "theoretical_loss": 4.085946776031922, + "tokens_seen": 340459520 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452938816449348, + "loss": 2.9653, + "theoretical_loss": 4.085856397486237, + "tokens_seen": 340525056 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045292878635907726, + "loss": 2.9517, + "theoretical_loss": 4.085766041201964, + "tokens_seen": 340590592 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529187562688064, + "loss": 2.742, + "theoretical_loss": 4.085675707169336, + "tokens_seen": 340656128 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004529087261785356, + "loss": 2.8849, + "theoretical_loss": 4.085585395378595, + "tokens_seen": 340721664 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 199795, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8782248497009277, + "objective/train/theoretical_loss": 4.085495105819987, + "objective/train/tokens_used": 361247200, + "theoretical_loss": 4.085495105819987, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528986960882648, + "loss": 2.8611, + "theoretical_loss": 4.085495105819987, + "tokens_seen": 340787200 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452888665997994, + "loss": 3.0898, + "theoretical_loss": 4.085404838483767, + "tokens_seen": 340852736 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045287863590772316, + "loss": 2.9958, + "theoretical_loss": 4.085314593360194, + "tokens_seen": 340918272 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528686058174524, + "loss": 3.0597, + "theoretical_loss": 4.085224370439533, + "tokens_seen": 340983808 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528585757271815, + "loss": 2.7746, + "theoretical_loss": 4.085134169712055, + "tokens_seen": 341049344 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045284854563691076, + "loss": 2.9852, + "theoretical_loss": 4.085043991168039, + "tokens_seen": 341114880 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528385155466399, + "loss": 3.0912, + "theoretical_loss": 4.084953834797769, + "tokens_seen": 341180416 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528284854563691, + "loss": 2.96, + "theoretical_loss": 4.0848637005915345, + "tokens_seen": 341245952 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528184553660983, + "loss": 3.2266, + "theoretical_loss": 4.0847735885396315, + "tokens_seen": 341311488 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004528084252758275, + "loss": 2.9558, + "theoretical_loss": 4.084683498632363, + "tokens_seen": 341377024 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045279839518555667, + "loss": 3.0659, + "theoretical_loss": 4.084593430860037, + "tokens_seen": 341442560 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045278836509528585, + "loss": 2.8982, + "theoretical_loss": 4.084503385212968, + "tokens_seen": 341508096 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045277833500501503, + "loss": 2.7515, + "theoretical_loss": 4.084413361681476, + "tokens_seen": 341573632 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045276830491474426, + "loss": 3.1749, + "theoretical_loss": 4.084323360255889, + "tokens_seen": 341639168 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527582748244734, + "loss": 2.8382, + "theoretical_loss": 4.0842333809265385, + "tokens_seen": 341704704 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004527482447342026, + "loss": 2.8088, + "theoretical_loss": 4.084143423683765, + "tokens_seen": 341770240 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045273821464393175, + "loss": 2.8347, + "theoretical_loss": 4.0840534885179105, + "tokens_seen": 341835776 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452728184553661, + "loss": 3.0138, + "theoretical_loss": 4.083963575419329, + "tokens_seen": 341901312 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045271815446339017, + "loss": 2.9717, + "theoretical_loss": 4.083873684378376, + "tokens_seen": 341966848 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045270812437311935, + "loss": 2.9964, + "theoretical_loss": 4.083783815385415, + "tokens_seen": 342032384 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526980942828486, + "loss": 3.0922, + "theoretical_loss": 4.083693968430816, + "tokens_seen": 342097920 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045268806419257777, + "loss": 2.9981, + "theoretical_loss": 4.083604143504952, + "tokens_seen": 342163456 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045267803410230695, + "loss": 3.4607, + "theoretical_loss": 4.083514340598207, + "tokens_seen": 342228992 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045266800401203613, + "loss": 3.0061, + "theoretical_loss": 4.083424559700966, + "tokens_seen": 342294528 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526579739217653, + "loss": 2.9478, + "theoretical_loss": 4.083334800803624, + "tokens_seen": 342360064 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 200497, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0360288619995117, + "objective/train/theoretical_loss": 4.08324506389658, + "objective/train/tokens_used": 362885600, + "theoretical_loss": 4.08324506389658, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526479438314945, + "loss": 2.9394, + "theoretical_loss": 4.08324506389658, + "tokens_seen": 342425600 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526379137412237, + "loss": 2.9863, + "theoretical_loss": 4.083155348970238, + "tokens_seen": 342491136 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045262788365095285, + "loss": 2.9571, + "theoretical_loss": 4.083065656015012, + "tokens_seen": 342556672 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526178535606821, + "loss": 3.2024, + "theoretical_loss": 4.082975985021317, + "tokens_seen": 342622208 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004526078234704112, + "loss": 3.1355, + "theoretical_loss": 4.082886335979578, + "tokens_seen": 342687744 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045259779338014045, + "loss": 2.9816, + "theoretical_loss": 4.082796708880224, + "tokens_seen": 342753280 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045258776328986963, + "loss": 3.1751, + "theoretical_loss": 4.082707103713689, + "tokens_seen": 342818816 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525777331995988, + "loss": 2.8529, + "theoretical_loss": 4.082617520470418, + "tokens_seen": 342884352 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452567703109328, + "loss": 3.0098, + "theoretical_loss": 4.0825279591408545, + "tokens_seen": 342949888 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045255767301905723, + "loss": 2.8312, + "theoretical_loss": 4.082438419715455, + "tokens_seen": 343015424 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045254764292878636, + "loss": 2.8357, + "theoretical_loss": 4.082348902184677, + "tokens_seen": 343080960 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525376128385156, + "loss": 2.9867, + "theoretical_loss": 4.082259406538987, + "tokens_seen": 343146496 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004525275827482447, + "loss": 2.9535, + "theoretical_loss": 4.082169932768856, + "tokens_seen": 343212032 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045251755265797395, + "loss": 2.8995, + "theoretical_loss": 4.082080480864761, + "tokens_seen": 343277568 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045250752256770314, + "loss": 3.0748, + "theoretical_loss": 4.0819910508171855, + "tokens_seen": 343343104 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524974924774323, + "loss": 2.9816, + "theoretical_loss": 4.08190164261662, + "tokens_seen": 343408640 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524874623871615, + "loss": 2.9615, + "theoretical_loss": 4.081812256253557, + "tokens_seen": 343474176 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524774322968907, + "loss": 3.0754, + "theoretical_loss": 4.0817228917185, + "tokens_seen": 343539712 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045246740220661986, + "loss": 3.277, + "theoretical_loss": 4.081633549001955, + "tokens_seen": 343605248 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524573721163491, + "loss": 2.9695, + "theoretical_loss": 4.081544228094436, + "tokens_seen": 343670784 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524473420260782, + "loss": 3.0677, + "theoretical_loss": 4.08145492898646, + "tokens_seen": 343736320 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045243731193580746, + "loss": 3.0617, + "theoretical_loss": 4.081365651668555, + "tokens_seen": 343801856 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524272818455366, + "loss": 2.8337, + "theoretical_loss": 4.081276396131249, + "tokens_seen": 343867392 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004524172517552658, + "loss": 2.9259, + "theoretical_loss": 4.0811871623650795, + "tokens_seen": 343932928 + }, + { + "epoch": 0.1, + "learning_rate": 0.000452407221664995, + "loss": 3.2243, + "theoretical_loss": 4.08109795036059, + "tokens_seen": 343998464 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 201708, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.375955820083618, + "objective/train/theoretical_loss": 4.081008760108329, + "objective/train/tokens_used": 364524000, + "theoretical_loss": 4.081008760108329, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523971915747242, + "loss": 3.0911, + "theoretical_loss": 4.081008760108329, + "tokens_seen": 344064000 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045238716148445336, + "loss": 2.8323, + "theoretical_loss": 4.08091959159885, + "tokens_seen": 344129536 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523771313941826, + "loss": 3.0438, + "theoretical_loss": 4.0808304448227135, + "tokens_seen": 344195072 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523671013039117, + "loss": 3.0118, + "theoretical_loss": 4.080741319770486, + "tokens_seen": 344260608 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045235707121364096, + "loss": 3.1077, + "theoretical_loss": 4.080652216432741, + "tokens_seen": 344326144 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523470411233701, + "loss": 2.9864, + "theoretical_loss": 4.0805631348000535, + "tokens_seen": 344391680 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523370110330993, + "loss": 2.7933, + "theoretical_loss": 4.080474074863011, + "tokens_seen": 344457216 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523269809428285, + "loss": 3.0455, + "theoretical_loss": 4.0803850366122, + "tokens_seen": 344522752 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004523169508525577, + "loss": 3.0973, + "theoretical_loss": 4.080296020038219, + "tokens_seen": 344588288 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045230692076228687, + "loss": 2.8627, + "theoretical_loss": 4.080207025131667, + "tokens_seen": 344653824 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045229689067201605, + "loss": 2.7906, + "theoretical_loss": 4.0801180518831535, + "tokens_seen": 344719360 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045228686058174523, + "loss": 2.7051, + "theoretical_loss": 4.080029100283291, + "tokens_seen": 344784896 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045227683049147446, + "loss": 3.1299, + "theoretical_loss": 4.079940170322698, + "tokens_seen": 344850432 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522668004012036, + "loss": 2.7122, + "theoretical_loss": 4.079851261992, + "tokens_seen": 344915968 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522567703109328, + "loss": 2.7792, + "theoretical_loss": 4.0797623752818275, + "tokens_seen": 344981504 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045224674022066195, + "loss": 3.1825, + "theoretical_loss": 4.079673510182818, + "tokens_seen": 345047040 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004522367101303912, + "loss": 2.8207, + "theoretical_loss": 4.079584666685612, + "tokens_seen": 345112576 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045222668004012037, + "loss": 2.9957, + "theoretical_loss": 4.07949584478086, + "tokens_seen": 345178112 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045221664994984955, + "loss": 2.8706, + "theoretical_loss": 4.079407044459216, + "tokens_seen": 345243648 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045220661985957873, + "loss": 2.7394, + "theoretical_loss": 4.079318265711338, + "tokens_seen": 345309184 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045219658976930797, + "loss": 2.7548, + "theoretical_loss": 4.0792295085278925, + "tokens_seen": 345374720 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521865596790371, + "loss": 2.9065, + "theoretical_loss": 4.079140772899552, + "tokens_seen": 345440256 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045217652958876633, + "loss": 3.0047, + "theoretical_loss": 4.079052058816993, + "tokens_seen": 345505792 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045216649949849546, + "loss": 3.1107, + "theoretical_loss": 4.078963366270899, + "tokens_seen": 345571328 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521564694082247, + "loss": 2.7583, + "theoretical_loss": 4.078874695251959, + "tokens_seen": 345636864 + }, + { + "epoch": 0.1, + "objective/train/docs_used": 202216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1073813438415527, + "objective/train/theoretical_loss": 4.078786045750867, + "objective/train/tokens_used": 366162400, + "theoretical_loss": 4.078786045750867, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045214643931795387, + "loss": 2.8222, + "theoretical_loss": 4.078786045750867, + "tokens_seen": 345702400 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045213640922768305, + "loss": 3.0401, + "theoretical_loss": 4.078697417758325, + "tokens_seen": 345767936 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045212637913741223, + "loss": 3.0623, + "theoretical_loss": 4.078608811265037, + "tokens_seen": 345833472 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521163490471414, + "loss": 2.8066, + "theoretical_loss": 4.078520226261718, + "tokens_seen": 345899008 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004521063189568706, + "loss": 2.8332, + "theoretical_loss": 4.078431662739083, + "tokens_seen": 345964544 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045209628886659983, + "loss": 3.0442, + "theoretical_loss": 4.078343120687858, + "tokens_seen": 346030080 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045208625877632896, + "loss": 3.0793, + "theoretical_loss": 4.078254600098771, + "tokens_seen": 346095616 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520762286860582, + "loss": 3.2073, + "theoretical_loss": 4.078166100962558, + "tokens_seen": 346161152 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520661985957873, + "loss": 2.9417, + "theoretical_loss": 4.078077623269959, + "tokens_seen": 346226688 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045205616850551656, + "loss": 2.9708, + "theoretical_loss": 4.0779891670117205, + "tokens_seen": 346292224 + }, + { + "epoch": 0.1, + "learning_rate": 0.00045204613841524574, + "loss": 3.0795, + "theoretical_loss": 4.077900732178595, + "tokens_seen": 346357760 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520361083249749, + "loss": 3.1212, + "theoretical_loss": 4.0778123187613415, + "tokens_seen": 346423296 + }, + { + "epoch": 0.1, + "learning_rate": 0.0004520260782347041, + "loss": 3.0503, + "theoretical_loss": 4.077723926750724, + "tokens_seen": 346488832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045201604814443334, + "loss": 2.8794, + "theoretical_loss": 4.07763555613751, + "tokens_seen": 346554368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045200601805416246, + "loss": 3.0506, + "theoretical_loss": 4.077547206912477, + "tokens_seen": 346619904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519959879638917, + "loss": 3.0215, + "theoretical_loss": 4.077458879066406, + "tokens_seen": 346685440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519859578736208, + "loss": 2.9088, + "theoretical_loss": 4.077370572590082, + "tokens_seen": 346750976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045197592778335006, + "loss": 2.8665, + "theoretical_loss": 4.077282287474299, + "tokens_seen": 346816512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045196589769307924, + "loss": 3.084, + "theoretical_loss": 4.077194023709854, + "tokens_seen": 346882048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519558676028084, + "loss": 2.9599, + "theoretical_loss": 4.077105781287552, + "tokens_seen": 346947584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045194583751253766, + "loss": 2.9752, + "theoretical_loss": 4.077017560198202, + "tokens_seen": 347013120 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519358074222668, + "loss": 3.182, + "theoretical_loss": 4.0769293604326196, + "tokens_seen": 347078656 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451925777331996, + "loss": 2.9303, + "theoretical_loss": 4.076841181981624, + "tokens_seen": 347144192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519157472417252, + "loss": 3.0146, + "theoretical_loss": 4.076753024836044, + "tokens_seen": 347209728 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004519057171514544, + "loss": 3.2817, + "theoretical_loss": 4.076664888986713, + "tokens_seen": 347275264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 203448, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4970381259918213, + "objective/train/theoretical_loss": 4.076576774424465, + "objective/train/tokens_used": 367800800, + "theoretical_loss": 4.076576774424465, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045189568706118356, + "loss": 2.8287, + "theoretical_loss": 4.076576774424465, + "tokens_seen": 347340800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518856569709128, + "loss": 2.8508, + "theoretical_loss": 4.0764886811401455, + "tokens_seen": 347406336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518756268806419, + "loss": 2.837, + "theoretical_loss": 4.0764006091246054, + "tokens_seen": 347471872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045186559679037116, + "loss": 3.1222, + "theoretical_loss": 4.076312558368698, + "tokens_seen": 347537408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518555667001003, + "loss": 2.8597, + "theoretical_loss": 4.076224528863284, + "tokens_seen": 347602944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518455366098295, + "loss": 2.9747, + "theoretical_loss": 4.0761365205992295, + "tokens_seen": 347668480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518355065195587, + "loss": 2.8766, + "theoretical_loss": 4.076048533567406, + "tokens_seen": 347734016 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004518254764292879, + "loss": 3.0041, + "theoretical_loss": 4.075960567758694, + "tokens_seen": 347799552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045181544633901707, + "loss": 2.9555, + "theoretical_loss": 4.075872623163973, + "tokens_seen": 347865088 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045180541624874625, + "loss": 2.7585, + "theoretical_loss": 4.075784699774134, + "tokens_seen": 347930624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045179538615847543, + "loss": 2.9776, + "theoretical_loss": 4.075696797580071, + "tokens_seen": 347996160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045178535606820466, + "loss": 3.0587, + "theoretical_loss": 4.075608916572684, + "tokens_seen": 348061696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517753259779338, + "loss": 2.9124, + "theoretical_loss": 4.075521056742879, + "tokens_seen": 348127232 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451765295887663, + "loss": 3.3636, + "theoretical_loss": 4.0754332180815664, + "tokens_seen": 348192768 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045175526579739215, + "loss": 3.1258, + "theoretical_loss": 4.075345400579664, + "tokens_seen": 348258304 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004517452357071214, + "loss": 2.8544, + "theoretical_loss": 4.075257604228094, + "tokens_seen": 348323840 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045173520561685057, + "loss": 2.979, + "theoretical_loss": 4.075169829017785, + "tokens_seen": 348389376 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045172517552657975, + "loss": 2.9919, + "theoretical_loss": 4.07508207493967, + "tokens_seen": 348454912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045171514543630893, + "loss": 3.0188, + "theoretical_loss": 4.074994341984689, + "tokens_seen": 348520448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045170511534603817, + "loss": 2.7695, + "theoretical_loss": 4.074906630143786, + "tokens_seen": 348585984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516950852557673, + "loss": 3.2483, + "theoretical_loss": 4.074818939407912, + "tokens_seen": 348651520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045168505516549653, + "loss": 3.1331, + "theoretical_loss": 4.074731269768024, + "tokens_seen": 348717056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045167502507522566, + "loss": 2.8828, + "theoretical_loss": 4.074643621215082, + "tokens_seen": 348782592 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516649949849549, + "loss": 2.8352, + "theoretical_loss": 4.074555993740055, + "tokens_seen": 348848128 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045165496489468407, + "loss": 3.1457, + "theoretical_loss": 4.074468387333914, + "tokens_seen": 348913664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 203975, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.054220199584961, + "objective/train/theoretical_loss": 4.074380801987639, + "objective/train/tokens_used": 369439200, + "theoretical_loss": 4.074380801987639, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045164493480441325, + "loss": 3.0341, + "theoretical_loss": 4.074380801987639, + "tokens_seen": 348979200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045163490471414244, + "loss": 3.2664, + "theoretical_loss": 4.074293237692212, + "tokens_seen": 349044736 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516248746238716, + "loss": 3.0151, + "theoretical_loss": 4.074205694438623, + "tokens_seen": 349110272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004516148445336008, + "loss": 3.0995, + "theoretical_loss": 4.074118172217869, + "tokens_seen": 349175808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045160481444333003, + "loss": 2.9087, + "theoretical_loss": 4.074030671020948, + "tokens_seen": 349241344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045159478435305916, + "loss": 2.9197, + "theoretical_loss": 4.073943190838868, + "tokens_seen": 349306880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515847542627884, + "loss": 2.9025, + "theoretical_loss": 4.073855731662639, + "tokens_seen": 349372416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515747241725175, + "loss": 3.0998, + "theoretical_loss": 4.07376829348328, + "tokens_seen": 349437952 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045156469408224676, + "loss": 3.1268, + "theoretical_loss": 4.073680876291811, + "tokens_seen": 349503488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045155466399197594, + "loss": 2.8534, + "theoretical_loss": 4.073593480079262, + "tokens_seen": 349569024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515446339017051, + "loss": 2.983, + "theoretical_loss": 4.073506104836667, + "tokens_seen": 349634560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515346038114343, + "loss": 3.0149, + "theoretical_loss": 4.073418750555065, + "tokens_seen": 349700096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045152457372116354, + "loss": 3.0564, + "theoretical_loss": 4.073331417225501, + "tokens_seen": 349765632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045151454363089266, + "loss": 2.986, + "theoretical_loss": 4.0732441048390235, + "tokens_seen": 349831168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004515045135406219, + "loss": 2.9123, + "theoretical_loss": 4.07315681338669, + "tokens_seen": 349896704 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451494483450351, + "loss": 2.8782, + "theoretical_loss": 4.07306954285956, + "tokens_seen": 349962240 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045148445336008026, + "loss": 2.8731, + "theoretical_loss": 4.072982293248703, + "tokens_seen": 350027776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045147442326980944, + "loss": 2.9387, + "theoretical_loss": 4.072895064545189, + "tokens_seen": 350093312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514643931795386, + "loss": 2.9963, + "theoretical_loss": 4.072807856740095, + "tokens_seen": 350158848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514543630892678, + "loss": 3.0698, + "theoretical_loss": 4.072720669824506, + "tokens_seen": 350224384 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451444332998997, + "loss": 2.9614, + "theoretical_loss": 4.072633503789511, + "tokens_seen": 350289920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045143430290872617, + "loss": 3.2203, + "theoretical_loss": 4.072546358626202, + "tokens_seen": 350355456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004514242728184554, + "loss": 3.0721, + "theoretical_loss": 4.07245923432568, + "tokens_seen": 350420992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045141424272818453, + "loss": 2.6751, + "theoretical_loss": 4.0723721308790495, + "tokens_seen": 350486528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045140421263791376, + "loss": 2.6701, + "theoretical_loss": 4.072285048277421, + "tokens_seen": 350552064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 205216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9308574199676514, + "objective/train/theoretical_loss": 4.072197986511911, + "objective/train/tokens_used": 371077600, + "theoretical_loss": 4.072197986511911, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045139418254764294, + "loss": 3.0665, + "theoretical_loss": 4.072197986511911, + "tokens_seen": 350617600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513841524573721, + "loss": 3.2069, + "theoretical_loss": 4.07211094557364, + "tokens_seen": 350683136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513741223671013, + "loss": 2.7969, + "theoretical_loss": 4.072023925453736, + "tokens_seen": 350748672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513640922768305, + "loss": 3.0486, + "theoretical_loss": 4.07193692614333, + "tokens_seen": 350814208 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045135406218655967, + "loss": 2.8671, + "theoretical_loss": 4.0718499476335595, + "tokens_seen": 350879744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513440320962889, + "loss": 2.9117, + "theoretical_loss": 4.071762989915569, + "tokens_seen": 350945280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045133400200601803, + "loss": 3.066, + "theoretical_loss": 4.071676052980506, + "tokens_seen": 351010816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045132397191574727, + "loss": 3.1371, + "theoretical_loss": 4.071589136819525, + "tokens_seen": 351076352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004513139418254764, + "loss": 2.9283, + "theoretical_loss": 4.071502241423786, + "tokens_seen": 351141888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045130391173520563, + "loss": 3.0026, + "theoretical_loss": 4.071415366784452, + "tokens_seen": 351207424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512938816449348, + "loss": 2.7252, + "theoretical_loss": 4.071328512892694, + "tokens_seen": 351272960 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451283851554664, + "loss": 2.9749, + "theoretical_loss": 4.071241679739687, + "tokens_seen": 351338496 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045127382146439317, + "loss": 2.8857, + "theoretical_loss": 4.071154867316614, + "tokens_seen": 351404032 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045126379137412235, + "loss": 3.0538, + "theoretical_loss": 4.07106807561466, + "tokens_seen": 351469568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045125376128385153, + "loss": 3.0197, + "theoretical_loss": 4.0709813046250165, + "tokens_seen": 351535104 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045124373119358077, + "loss": 2.8479, + "theoretical_loss": 4.070894554338881, + "tokens_seen": 351600640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512337011033099, + "loss": 2.9532, + "theoretical_loss": 4.070807824747456, + "tokens_seen": 351666176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045122367101303913, + "loss": 2.9436, + "theoretical_loss": 4.07072111584195, + "tokens_seen": 351731712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512136409227683, + "loss": 2.8975, + "theoretical_loss": 4.070634427613575, + "tokens_seen": 351797248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004512036108324975, + "loss": 2.8211, + "theoretical_loss": 4.070547760053551, + "tokens_seen": 351862784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045119358074222673, + "loss": 2.8698, + "theoretical_loss": 4.070461113153101, + "tokens_seen": 351928320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045118355065195586, + "loss": 2.8615, + "theoretical_loss": 4.0703744869034555, + "tokens_seen": 351993856 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511735205616851, + "loss": 2.9633, + "theoretical_loss": 4.070287881295848, + "tokens_seen": 352059392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511634904714143, + "loss": 2.8897, + "theoretical_loss": 4.070201296321519, + "tokens_seen": 352124928 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045115346038114345, + "loss": 2.9223, + "theoretical_loss": 4.070114731971715, + "tokens_seen": 352190464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 206163, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.493535280227661, + "objective/train/theoretical_loss": 4.070028188237686, + "objective/train/tokens_used": 372716000, + "theoretical_loss": 4.070028188237686, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045114343029087264, + "loss": 2.8772, + "theoretical_loss": 4.070028188237686, + "tokens_seen": 352256000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004511334002006018, + "loss": 3.067, + "theoretical_loss": 4.069941665110687, + "tokens_seen": 352321536 + }, + { + "epoch": 0.11, + "learning_rate": 0.000451123370110331, + "loss": 2.9246, + "theoretical_loss": 4.06985516258198, + "tokens_seen": 352387072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045111334002006023, + "loss": 3.006, + "theoretical_loss": 4.069768680642833, + "tokens_seen": 352452608 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045110330992978936, + "loss": 2.9814, + "theoretical_loss": 4.069682219284518, + "tokens_seen": 352518144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510932798395186, + "loss": 3.105, + "theoretical_loss": 4.069595778498311, + "tokens_seen": 352583680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510832497492477, + "loss": 2.8652, + "theoretical_loss": 4.069509358275495, + "tokens_seen": 352649216 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045107321965897696, + "loss": 2.854, + "theoretical_loss": 4.069422958607359, + "tokens_seen": 352714752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045106318956870614, + "loss": 2.8667, + "theoretical_loss": 4.069336579485196, + "tokens_seen": 352780288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510531594784353, + "loss": 2.9462, + "theoretical_loss": 4.0692502209003045, + "tokens_seen": 352845824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510431293881645, + "loss": 2.9122, + "theoretical_loss": 4.0691638828439896, + "tokens_seen": 352911360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045103309929789374, + "loss": 2.9916, + "theoretical_loss": 4.069077565307558, + "tokens_seen": 352976896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045102306920762286, + "loss": 2.9217, + "theoretical_loss": 4.068991268282327, + "tokens_seen": 353042432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510130391173521, + "loss": 2.6557, + "theoretical_loss": 4.068904991759615, + "tokens_seen": 353107968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004510030090270812, + "loss": 2.8802, + "theoretical_loss": 4.068818735730747, + "tokens_seen": 353173504 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045099297893681046, + "loss": 3.0742, + "theoretical_loss": 4.068732500187053, + "tokens_seen": 353239040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045098294884653964, + "loss": 2.9343, + "theoretical_loss": 4.068646285119871, + "tokens_seen": 353304576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509729187562688, + "loss": 2.7554, + "theoretical_loss": 4.06856009052054, + "tokens_seen": 353370112 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450962888665998, + "loss": 3.0227, + "theoretical_loss": 4.068473916380406, + "tokens_seen": 353435648 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509528585757272, + "loss": 2.8967, + "theoretical_loss": 4.068387762690822, + "tokens_seen": 353501184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045094282848545637, + "loss": 2.9807, + "theoretical_loss": 4.068301629443143, + "tokens_seen": 353566720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004509327983951856, + "loss": 2.9679, + "theoretical_loss": 4.068215516628733, + "tokens_seen": 353632256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045092276830491473, + "loss": 2.9093, + "theoretical_loss": 4.068129424238958, + "tokens_seen": 353697792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045091273821464396, + "loss": 2.9453, + "theoretical_loss": 4.06804335226519, + "tokens_seen": 353763328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045090270812437314, + "loss": 2.6639, + "theoretical_loss": 4.067957300698807, + "tokens_seen": 353828864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 206742, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6503775119781494, + "objective/train/theoretical_loss": 4.067871269531193, + "objective/train/tokens_used": 374354400, + "theoretical_loss": 4.067871269531193, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508926780341023, + "loss": 2.8327, + "theoretical_loss": 4.067871269531193, + "tokens_seen": 353894400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508826479438315, + "loss": 2.6181, + "theoretical_loss": 4.0677852587537355, + "tokens_seen": 353959936 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508726178535607, + "loss": 3.0139, + "theoretical_loss": 4.067699268357828, + "tokens_seen": 354025472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045086258776328987, + "loss": 2.9987, + "theoretical_loss": 4.0676132983348685, + "tokens_seen": 354091008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508525576730191, + "loss": 3.1759, + "theoretical_loss": 4.067527348676262, + "tokens_seen": 354156544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045084252758274823, + "loss": 2.9902, + "theoretical_loss": 4.067441419373417, + "tokens_seen": 354222080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045083249749247747, + "loss": 2.9154, + "theoretical_loss": 4.067355510417749, + "tokens_seen": 354287616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004508224674022066, + "loss": 2.7929, + "theoretical_loss": 4.067269621800675, + "tokens_seen": 354353152 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045081243731193583, + "loss": 2.8677, + "theoretical_loss": 4.067183753513622, + "tokens_seen": 354418688 + }, + { + "epoch": 0.11, + "learning_rate": 0.000450802407221665, + "loss": 2.9784, + "theoretical_loss": 4.067097905548019, + "tokens_seen": 354484224 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507923771313942, + "loss": 3.0821, + "theoretical_loss": 4.067012077895301, + "tokens_seen": 354549760 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045078234704112337, + "loss": 2.7986, + "theoretical_loss": 4.066926270546909, + "tokens_seen": 354615296 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045077231695085255, + "loss": 2.842, + "theoretical_loss": 4.066840483494287, + "tokens_seen": 354680832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045076228686058173, + "loss": 2.8722, + "theoretical_loss": 4.066754716728888, + "tokens_seen": 354746368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045075225677031097, + "loss": 2.9914, + "theoretical_loss": 4.066668970242167, + "tokens_seen": 354811904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507422266800401, + "loss": 2.8883, + "theoretical_loss": 4.066583244025583, + "tokens_seen": 354877440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045073219658976933, + "loss": 2.8113, + "theoretical_loss": 4.066497538070605, + "tokens_seen": 354942976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507221664994985, + "loss": 2.591, + "theoretical_loss": 4.066411852368702, + "tokens_seen": 355008512 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507121364092277, + "loss": 2.7762, + "theoretical_loss": 4.066326186911353, + "tokens_seen": 355074048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004507021063189569, + "loss": 2.7745, + "theoretical_loss": 4.066240541690038, + "tokens_seen": 355139584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045069207622868606, + "loss": 2.7743, + "theoretical_loss": 4.066154916696245, + "tokens_seen": 355205120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045068204613841524, + "loss": 2.9515, + "theoretical_loss": 4.066069311921465, + "tokens_seen": 355270656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506720160481445, + "loss": 2.7834, + "theoretical_loss": 4.065983727357195, + "tokens_seen": 355336192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506619859578736, + "loss": 2.8959, + "theoretical_loss": 4.065898162994938, + "tokens_seen": 355401728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045065195586760284, + "loss": 2.8629, + "theoretical_loss": 4.065812618826202, + "tokens_seen": 355467264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 207983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6451644897460938, + "objective/train/theoretical_loss": 4.065727094842498, + "objective/train/tokens_used": 375992800, + "theoretical_loss": 4.065727094842498, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045064192577733196, + "loss": 2.8682, + "theoretical_loss": 4.065727094842498, + "tokens_seen": 355532800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506318956870612, + "loss": 2.7648, + "theoretical_loss": 4.0656415910353445, + "tokens_seen": 355598336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004506218655967904, + "loss": 2.9992, + "theoretical_loss": 4.065556107396265, + "tokens_seen": 355663872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045061183550651956, + "loss": 3.2104, + "theoretical_loss": 4.065470643916786, + "tokens_seen": 355729408 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045060180541624874, + "loss": 3.0774, + "theoretical_loss": 4.065385200588442, + "tokens_seen": 355794944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505917753259779, + "loss": 2.9544, + "theoretical_loss": 4.065299777402771, + "tokens_seen": 355860480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505817452357071, + "loss": 2.8752, + "theoretical_loss": 4.065214374351315, + "tokens_seen": 355926016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045057171514543634, + "loss": 2.7962, + "theoretical_loss": 4.065128991425624, + "tokens_seen": 355991552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045056168505516547, + "loss": 2.74, + "theoretical_loss": 4.06504362861725, + "tokens_seen": 356057088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505516549648947, + "loss": 2.8575, + "theoretical_loss": 4.064958285917752, + "tokens_seen": 356122624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505416248746239, + "loss": 2.8417, + "theoretical_loss": 4.064872963318696, + "tokens_seen": 356188160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045053159478435306, + "loss": 3.0414, + "theoretical_loss": 4.064787660811648, + "tokens_seen": 356253696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045052156469408224, + "loss": 2.7343, + "theoretical_loss": 4.064702378388183, + "tokens_seen": 356319232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505115346038114, + "loss": 3.0008, + "theoretical_loss": 4.064617116039878, + "tokens_seen": 356384768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004505015045135406, + "loss": 2.9624, + "theoretical_loss": 4.064531873758321, + "tokens_seen": 356450304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045049147442326984, + "loss": 2.9619, + "theoretical_loss": 4.064446651535098, + "tokens_seen": 356515840 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045048144433299897, + "loss": 2.7413, + "theoretical_loss": 4.0643614493618045, + "tokens_seen": 356581376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504714142427282, + "loss": 2.9723, + "theoretical_loss": 4.0642762672300385, + "tokens_seen": 356646912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045046138415245733, + "loss": 2.8596, + "theoretical_loss": 4.064191105131406, + "tokens_seen": 356712448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045045135406218657, + "loss": 2.7957, + "theoretical_loss": 4.064105963057515, + "tokens_seen": 356777984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504413239719158, + "loss": 2.7377, + "theoretical_loss": 4.064020840999981, + "tokens_seen": 356843520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045043129388164493, + "loss": 2.8931, + "theoretical_loss": 4.063935738950423, + "tokens_seen": 356909056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045042126379137416, + "loss": 3.1138, + "theoretical_loss": 4.063850656900465, + "tokens_seen": 356974592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045041123370110334, + "loss": 2.9516, + "theoretical_loss": 4.063765594841737, + "tokens_seen": 357040128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004504012036108325, + "loss": 2.9924, + "theoretical_loss": 4.063680552765874, + "tokens_seen": 357105664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 208569, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.472182512283325, + "objective/train/theoretical_loss": 4.063595530664515, + "objective/train/tokens_used": 377631200, + "theoretical_loss": 4.063595530664515, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503911735205617, + "loss": 2.9434, + "theoretical_loss": 4.063595530664515, + "tokens_seen": 357171200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503811434302909, + "loss": 2.7141, + "theoretical_loss": 4.063510528529306, + "tokens_seen": 357236736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045037111334002007, + "loss": 2.9001, + "theoretical_loss": 4.063425546351894, + "tokens_seen": 357302272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503610832497493, + "loss": 2.9435, + "theoretical_loss": 4.063340584123936, + "tokens_seen": 357367808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045035105315947843, + "loss": 3.0876, + "theoretical_loss": 4.063255641837092, + "tokens_seen": 357433344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045034102306920767, + "loss": 2.7518, + "theoretical_loss": 4.063170719483025, + "tokens_seen": 357498880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503309929789368, + "loss": 2.7653, + "theoretical_loss": 4.063085817053405, + "tokens_seen": 357564416 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045032096288866603, + "loss": 2.9062, + "theoretical_loss": 4.063000934539909, + "tokens_seen": 357629952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503109327983952, + "loss": 2.7442, + "theoretical_loss": 4.062916071934215, + "tokens_seen": 357695488 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004503009027081244, + "loss": 3.0829, + "theoretical_loss": 4.062831229228007, + "tokens_seen": 357761024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045029087261785357, + "loss": 2.8354, + "theoretical_loss": 4.062746406412977, + "tokens_seen": 357826560 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045028084252758275, + "loss": 2.9435, + "theoretical_loss": 4.062661603480818, + "tokens_seen": 357892096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045027081243731193, + "loss": 2.9386, + "theoretical_loss": 4.06257682042323, + "tokens_seen": 357957632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045026078234704117, + "loss": 2.8088, + "theoretical_loss": 4.0624920572319185, + "tokens_seen": 358023168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502507522567703, + "loss": 2.9035, + "theoretical_loss": 4.062407313898593, + "tokens_seen": 358088704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045024072216649953, + "loss": 2.8364, + "theoretical_loss": 4.062322590414967, + "tokens_seen": 358154240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502306920762287, + "loss": 2.8049, + "theoretical_loss": 4.0622378867727615, + "tokens_seen": 358219776 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502206619859579, + "loss": 2.9569, + "theoretical_loss": 4.062153202963701, + "tokens_seen": 358285312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004502106318956871, + "loss": 2.8268, + "theoretical_loss": 4.062068538979515, + "tokens_seen": 358350848 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045020060180541626, + "loss": 2.8442, + "theoretical_loss": 4.061983894811938, + "tokens_seen": 358416384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045019057171514544, + "loss": 3.1648, + "theoretical_loss": 4.0618992704527095, + "tokens_seen": 358481920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501805416248747, + "loss": 2.9194, + "theoretical_loss": 4.061814665893573, + "tokens_seen": 358547456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501705115346038, + "loss": 2.9655, + "theoretical_loss": 4.06173008112628, + "tokens_seen": 358612992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045016048144433304, + "loss": 2.8605, + "theoretical_loss": 4.061645516142583, + "tokens_seen": 358678528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045015045135406216, + "loss": 2.7901, + "theoretical_loss": 4.0615609709342415, + "tokens_seen": 358744064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 209910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.162785768508911, + "objective/train/theoretical_loss": 4.061476445493021, + "objective/train/tokens_used": 379269600, + "theoretical_loss": 4.061476445493021, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501404212637914, + "loss": 2.6388, + "theoretical_loss": 4.061476445493021, + "tokens_seen": 358809600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501303911735206, + "loss": 2.8801, + "theoretical_loss": 4.06139193981069, + "tokens_seen": 358875136 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045012036108324976, + "loss": 3.0025, + "theoretical_loss": 4.061307453879023, + "tokens_seen": 358940672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045011033099297894, + "loss": 2.7736, + "theoretical_loss": 4.061222987689797, + "tokens_seen": 359006208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004501003009027081, + "loss": 2.8362, + "theoretical_loss": 4.061138541234799, + "tokens_seen": 359071744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500902708124373, + "loss": 2.6216, + "theoretical_loss": 4.061054114505816, + "tokens_seen": 359137280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045008024072216654, + "loss": 3.0416, + "theoretical_loss": 4.060969707494642, + "tokens_seen": 359202816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045007021063189567, + "loss": 2.8598, + "theoretical_loss": 4.0608853201930755, + "tokens_seen": 359268352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500601805416249, + "loss": 3.0376, + "theoretical_loss": 4.0608009525929205, + "tokens_seen": 359333888 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500501504513541, + "loss": 2.9702, + "theoretical_loss": 4.060716604685986, + "tokens_seen": 359399424 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045004012036108326, + "loss": 2.911, + "theoretical_loss": 4.0606322764640845, + "tokens_seen": 359464960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045003009027081244, + "loss": 2.6158, + "theoretical_loss": 4.060547967919034, + "tokens_seen": 359530496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500200601805416, + "loss": 2.8483, + "theoretical_loss": 4.06046367904266, + "tokens_seen": 359596032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004500100300902708, + "loss": 2.8765, + "theoretical_loss": 4.060379409826788, + "tokens_seen": 359661568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00045000000000000004, + "loss": 2.9966, + "theoretical_loss": 4.0602951602632515, + "tokens_seen": 359727104 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044998996990972917, + "loss": 2.9513, + "theoretical_loss": 4.060210930343889, + "tokens_seen": 359792640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499799398194584, + "loss": 3.0588, + "theoretical_loss": 4.060126720060543, + "tokens_seen": 359858176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044996990972918753, + "loss": 2.4906, + "theoretical_loss": 4.060042529405061, + "tokens_seen": 359923712 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044995987963891677, + "loss": 2.9925, + "theoretical_loss": 4.059958358369295, + "tokens_seen": 359989248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044994984954864595, + "loss": 2.7972, + "theoretical_loss": 4.0598742069451035, + "tokens_seen": 360054784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044993981945837513, + "loss": 3.0398, + "theoretical_loss": 4.059790075124348, + "tokens_seen": 360120320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004499297893681043, + "loss": 2.7946, + "theoretical_loss": 4.059705962898896, + "tokens_seen": 360185856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044991975927783355, + "loss": 2.9623, + "theoretical_loss": 4.059621870260618, + "tokens_seen": 360251392 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044990972918756267, + "loss": 2.9151, + "theoretical_loss": 4.059537797201392, + "tokens_seen": 360316928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498996990972919, + "loss": 2.7803, + "theoretical_loss": 4.059453743713099, + "tokens_seen": 360382464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 210586, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.735008478164673, + "objective/train/theoretical_loss": 4.059369709787625, + "objective/train/tokens_used": 380908000, + "theoretical_loss": 4.059369709787625, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044988966900702103, + "loss": 2.9449, + "theoretical_loss": 4.059369709787625, + "tokens_seen": 360448000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044987963891675027, + "loss": 2.8092, + "theoretical_loss": 4.059285695416863, + "tokens_seen": 360513536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044986960882647945, + "loss": 2.7777, + "theoretical_loss": 4.059201700592707, + "tokens_seen": 360579072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044985957873620863, + "loss": 3.0467, + "theoretical_loss": 4.05911772530706, + "tokens_seen": 360644608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498495486459378, + "loss": 3.1316, + "theoretical_loss": 4.059033769551827, + "tokens_seen": 360710144 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449839518555667, + "loss": 2.8016, + "theoretical_loss": 4.058949833318917, + "tokens_seen": 360775680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498294884653962, + "loss": 3.0944, + "theoretical_loss": 4.058865916600248, + "tokens_seen": 360841216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004498194583751254, + "loss": 2.8891, + "theoretical_loss": 4.058782019387738, + "tokens_seen": 360906752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044980942828485454, + "loss": 2.8118, + "theoretical_loss": 4.058698141673313, + "tokens_seen": 360972288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044979939819458377, + "loss": 2.732, + "theoretical_loss": 4.058614283448903, + "tokens_seen": 361037824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497893681043129, + "loss": 2.844, + "theoretical_loss": 4.0585304447064425, + "tokens_seen": 361103360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044977933801404213, + "loss": 2.8023, + "theoretical_loss": 4.05844662543787, + "tokens_seen": 361168896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497693079237713, + "loss": 2.8618, + "theoretical_loss": 4.058362825635132, + "tokens_seen": 361234432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497592778335005, + "loss": 2.9652, + "theoretical_loss": 4.058279045290176, + "tokens_seen": 361299968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497492477432297, + "loss": 2.7903, + "theoretical_loss": 4.058195284394955, + "tokens_seen": 361365504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497392176529589, + "loss": 2.995, + "theoretical_loss": 4.058111542941429, + "tokens_seen": 361431040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044972918756268804, + "loss": 2.835, + "theoretical_loss": 4.05802782092156, + "tokens_seen": 361496576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497191574724173, + "loss": 2.7238, + "theoretical_loss": 4.057944118327319, + "tokens_seen": 361562112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004497091273821464, + "loss": 3.148, + "theoretical_loss": 4.057860435150675, + "tokens_seen": 361627648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044969909729187564, + "loss": 2.6922, + "theoretical_loss": 4.057776771383609, + "tokens_seen": 361693184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496890672016049, + "loss": 2.869, + "theoretical_loss": 4.057693127018101, + "tokens_seen": 361758720 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449679037111334, + "loss": 3.0336, + "theoretical_loss": 4.057609502046141, + "tokens_seen": 361824256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044966900702106324, + "loss": 3.0169, + "theoretical_loss": 4.057525896459719, + "tokens_seen": 361889792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044965897693079236, + "loss": 2.8861, + "theoretical_loss": 4.057442310250831, + "tokens_seen": 361955328 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496489468405216, + "loss": 2.9571, + "theoretical_loss": 4.0573587434114815, + "tokens_seen": 362020864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 211693, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5061984062194824, + "objective/train/theoretical_loss": 4.057275195933674, + "objective/train/tokens_used": 382546400, + "theoretical_loss": 4.057275195933674, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496389167502508, + "loss": 3.2087, + "theoretical_loss": 4.057275195933674, + "tokens_seen": 362086400 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044962888665997996, + "loss": 2.835, + "theoretical_loss": 4.057191667809421, + "tokens_seen": 362151936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044961885656970914, + "loss": 2.8273, + "theoretical_loss": 4.057108159030737, + "tokens_seen": 362217472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004496088264794383, + "loss": 2.9131, + "theoretical_loss": 4.057024669589644, + "tokens_seen": 362283008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495987963891675, + "loss": 2.7972, + "theoretical_loss": 4.056941199478166, + "tokens_seen": 362348544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044958876629889674, + "loss": 3.1665, + "theoretical_loss": 4.0568577486883335, + "tokens_seen": 362414080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044957873620862587, + "loss": 2.9409, + "theoretical_loss": 4.056774317212181, + "tokens_seen": 362479616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495687061183551, + "loss": 2.6666, + "theoretical_loss": 4.056690905041748, + "tokens_seen": 362545152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495586760280843, + "loss": 2.8504, + "theoretical_loss": 4.056607512169078, + "tokens_seen": 362610688 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044954864593781346, + "loss": 2.8855, + "theoretical_loss": 4.056524138586221, + "tokens_seen": 362676224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044953861584754264, + "loss": 2.654, + "theoretical_loss": 4.056440784285231, + "tokens_seen": 362741760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004495285857572718, + "loss": 2.9835, + "theoretical_loss": 4.056357449258163, + "tokens_seen": 362807296 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449518555667001, + "loss": 2.9323, + "theoretical_loss": 4.056274133497084, + "tokens_seen": 362872832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044950852557673024, + "loss": 2.8978, + "theoretical_loss": 4.056190836994059, + "tokens_seen": 362938368 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044949849548645937, + "loss": 2.6167, + "theoretical_loss": 4.056107559741161, + "tokens_seen": 363003904 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494884653961886, + "loss": 2.942, + "theoretical_loss": 4.056024301730467, + "tokens_seen": 363069440 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044947843530591773, + "loss": 2.9565, + "theoretical_loss": 4.05594106295406, + "tokens_seen": 363134976 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044946840521564697, + "loss": 2.7726, + "theoretical_loss": 4.055857843404025, + "tokens_seen": 363200512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044945837512537615, + "loss": 3.0216, + "theoretical_loss": 4.0557746430724535, + "tokens_seen": 363266048 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044944834503510533, + "loss": 3.1954, + "theoretical_loss": 4.05569146195144, + "tokens_seen": 363331584 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494383149448345, + "loss": 2.9553, + "theoretical_loss": 4.055608300033088, + "tokens_seen": 363397120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044942828485456375, + "loss": 3.024, + "theoretical_loss": 4.0555251573095, + "tokens_seen": 363462656 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044941825476429287, + "loss": 2.7395, + "theoretical_loss": 4.0554420337727874, + "tokens_seen": 363528192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004494082246740221, + "loss": 2.8736, + "theoretical_loss": 4.055358929415063, + "tokens_seen": 363593728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044939819458375123, + "loss": 2.8492, + "theoretical_loss": 4.055275844228448, + "tokens_seen": 363659264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 212305, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9765894412994385, + "objective/train/theoretical_loss": 4.055192778205064, + "objective/train/tokens_used": 384184800, + "theoretical_loss": 4.055192778205064, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044938816449348047, + "loss": 3.0477, + "theoretical_loss": 4.055192778205064, + "tokens_seen": 363724800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044937813440320965, + "loss": 2.7255, + "theoretical_loss": 4.0551097313370414, + "tokens_seen": 363790336 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044936810431293883, + "loss": 2.974, + "theoretical_loss": 4.055026703616512, + "tokens_seen": 363855872 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449358074222668, + "loss": 2.9851, + "theoretical_loss": 4.054943695035614, + "tokens_seen": 363921408 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493480441323972, + "loss": 3.114, + "theoretical_loss": 4.054860705586489, + "tokens_seen": 363986944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493380140421264, + "loss": 2.9801, + "theoretical_loss": 4.054777735261286, + "tokens_seen": 364052480 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004493279839518556, + "loss": 2.8191, + "theoretical_loss": 4.054694784052155, + "tokens_seen": 364118016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044931795386158474, + "loss": 2.8661, + "theoretical_loss": 4.054611851951253, + "tokens_seen": 364183552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044930792377131397, + "loss": 2.7798, + "theoretical_loss": 4.054528938950741, + "tokens_seen": 364249088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492978936810431, + "loss": 2.9318, + "theoretical_loss": 4.054446045042783, + "tokens_seen": 364314624 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044928786359077234, + "loss": 3.0197, + "theoretical_loss": 4.054363170219551, + "tokens_seen": 364380160 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492778335005015, + "loss": 2.8279, + "theoretical_loss": 4.054280314473219, + "tokens_seen": 364445696 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492678034102307, + "loss": 2.8172, + "theoretical_loss": 4.054197477795967, + "tokens_seen": 364511232 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492577733199599, + "loss": 3.0234, + "theoretical_loss": 4.0541146601799785, + "tokens_seen": 364576768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492477432296891, + "loss": 2.675, + "theoretical_loss": 4.0540318616174424, + "tokens_seen": 364642304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044923771313941824, + "loss": 2.6453, + "theoretical_loss": 4.0539490821005515, + "tokens_seen": 364707840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492276830491475, + "loss": 2.7324, + "theoretical_loss": 4.053866321621504, + "tokens_seen": 364773376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004492176529588766, + "loss": 2.8083, + "theoretical_loss": 4.053783580172501, + "tokens_seen": 364838912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044920762286860584, + "loss": 2.8838, + "theoretical_loss": 4.053700857745753, + "tokens_seen": 364904448 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449197592778335, + "loss": 2.492, + "theoretical_loss": 4.053618154333467, + "tokens_seen": 364969984 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491875626880642, + "loss": 2.869, + "theoretical_loss": 4.053535469927864, + "tokens_seen": 365035520 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491775325977934, + "loss": 2.8246, + "theoretical_loss": 4.05345280452116, + "tokens_seen": 365101056 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044916750250752256, + "loss": 2.7243, + "theoretical_loss": 4.053370158105584, + "tokens_seen": 365166592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044915747241725174, + "loss": 2.5572, + "theoretical_loss": 4.053287530673364, + "tokens_seen": 365232128 + }, + { + "epoch": 0.11, + "learning_rate": 0.000449147442326981, + "loss": 2.9422, + "theoretical_loss": 4.053204922216736, + "tokens_seen": 365297664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 213520, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0496366024017334, + "objective/train/theoretical_loss": 4.053122332727937, + "objective/train/tokens_used": 385823200, + "theoretical_loss": 4.053122332727937, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491374122367101, + "loss": 2.9336, + "theoretical_loss": 4.053122332727937, + "tokens_seen": 365363200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044912738214643934, + "loss": 2.6247, + "theoretical_loss": 4.053039762199212, + "tokens_seen": 365428736 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044911735205616847, + "loss": 2.6713, + "theoretical_loss": 4.05295721062281, + "tokens_seen": 365494272 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004491073219658977, + "loss": 3.0444, + "theoretical_loss": 4.0528746779909826, + "tokens_seen": 365559808 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490972918756269, + "loss": 2.8455, + "theoretical_loss": 4.052792164295987, + "tokens_seen": 365625344 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044908726178535607, + "loss": 2.7943, + "theoretical_loss": 4.052709669530085, + "tokens_seen": 365690880 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044907723169508525, + "loss": 2.8501, + "theoretical_loss": 4.052627193685544, + "tokens_seen": 365756416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490672016048145, + "loss": 2.8558, + "theoretical_loss": 4.052544736754634, + "tokens_seen": 365821952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490571715145436, + "loss": 2.9659, + "theoretical_loss": 4.0524622987296315, + "tokens_seen": 365887488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044904714142427284, + "loss": 2.8449, + "theoretical_loss": 4.052379879602816, + "tokens_seen": 365953024 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044903711133400197, + "loss": 2.8962, + "theoretical_loss": 4.052297479366471, + "tokens_seen": 366018560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490270812437312, + "loss": 2.6586, + "theoretical_loss": 4.052215098012887, + "tokens_seen": 366084096 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004490170511534604, + "loss": 2.6115, + "theoretical_loss": 4.052132735534358, + "tokens_seen": 366149632 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044900702106318957, + "loss": 2.5501, + "theoretical_loss": 4.0520503919231805, + "tokens_seen": 366215168 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044899699097291875, + "loss": 2.7229, + "theoretical_loss": 4.051968067171659, + "tokens_seen": 366280704 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044898696088264793, + "loss": 2.66, + "theoretical_loss": 4.051885761272098, + "tokens_seen": 366346240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489769307923771, + "loss": 2.8702, + "theoretical_loss": 4.051803474216813, + "tokens_seen": 366411776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044896690070210635, + "loss": 3.0329, + "theoretical_loss": 4.051721205998117, + "tokens_seen": 366477312 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489568706118355, + "loss": 3.0362, + "theoretical_loss": 4.051638956608333, + "tokens_seen": 366542848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489468405215647, + "loss": 3.0241, + "theoretical_loss": 4.0515567260397845, + "tokens_seen": 366608384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044893681043129395, + "loss": 3.1542, + "theoretical_loss": 4.051474514284801, + "tokens_seen": 366673920 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044892678034102307, + "loss": 2.9845, + "theoretical_loss": 4.051392321335719, + "tokens_seen": 366739456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004489167502507523, + "loss": 2.8097, + "theoretical_loss": 4.051310147184875, + "tokens_seen": 366804992 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044890672016048143, + "loss": 2.7092, + "theoretical_loss": 4.0512279918246135, + "tokens_seen": 366870528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044889669007021067, + "loss": 2.7877, + "theoretical_loss": 4.051145855247281, + "tokens_seen": 366936064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 214095, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.887453317642212, + "objective/train/theoretical_loss": 4.051063737445231, + "objective/train/tokens_used": 387461600, + "theoretical_loss": 4.051063737445231, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044888665997993985, + "loss": 3.0162, + "theoretical_loss": 4.051063737445231, + "tokens_seen": 367001600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044887662988966903, + "loss": 2.7889, + "theoretical_loss": 4.0509816384108195, + "tokens_seen": 367067136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488665997993982, + "loss": 2.966, + "theoretical_loss": 4.050899558136408, + "tokens_seen": 367132672 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488565697091274, + "loss": 2.8294, + "theoretical_loss": 4.050817496614361, + "tokens_seen": 367198208 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488465396188566, + "loss": 2.916, + "theoretical_loss": 4.050735453837049, + "tokens_seen": 367263744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488365095285858, + "loss": 2.8996, + "theoretical_loss": 4.0506534297968475, + "tokens_seen": 367329280 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044882647943831494, + "loss": 2.8554, + "theoretical_loss": 4.050571424486135, + "tokens_seen": 367394816 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488164493480442, + "loss": 2.7958, + "theoretical_loss": 4.050489437897295, + "tokens_seen": 367460352 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004488064192577733, + "loss": 2.9325, + "theoretical_loss": 4.050407470022715, + "tokens_seen": 367525888 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044879638916750254, + "loss": 2.9501, + "theoretical_loss": 4.050325520854788, + "tokens_seen": 367591424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487863590772317, + "loss": 2.6799, + "theoretical_loss": 4.050243590385909, + "tokens_seen": 367656960 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487763289869609, + "loss": 2.9033, + "theoretical_loss": 4.0501616786084815, + "tokens_seen": 367722496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487662988966901, + "loss": 2.8158, + "theoretical_loss": 4.05007978551491, + "tokens_seen": 367788032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487562688064193, + "loss": 2.8058, + "theoretical_loss": 4.049997911097606, + "tokens_seen": 367853568 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044874623871614844, + "loss": 2.7876, + "theoretical_loss": 4.049916055348982, + "tokens_seen": 367919104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487362086258777, + "loss": 3.0632, + "theoretical_loss": 4.049834218261458, + "tokens_seen": 367984640 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487261785356068, + "loss": 3.0815, + "theoretical_loss": 4.049752399827458, + "tokens_seen": 368050176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044871614844533604, + "loss": 2.7292, + "theoretical_loss": 4.049670600039409, + "tokens_seen": 368115712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004487061183550652, + "loss": 2.8603, + "theoretical_loss": 4.049588818889744, + "tokens_seen": 368181248 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486960882647944, + "loss": 3.1734, + "theoretical_loss": 4.049507056370899, + "tokens_seen": 368246784 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486860581745236, + "loss": 3.0132, + "theoretical_loss": 4.049425312475315, + "tokens_seen": 368312320 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044867602808425276, + "loss": 2.9497, + "theoretical_loss": 4.049343587195439, + "tokens_seen": 368377856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044866599799398194, + "loss": 2.9889, + "theoretical_loss": 4.049261880523719, + "tokens_seen": 368443392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486559679037112, + "loss": 2.7858, + "theoretical_loss": 4.04918019245261, + "tokens_seen": 368508928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486459378134403, + "loss": 2.7187, + "theoretical_loss": 4.049098522974571, + "tokens_seen": 368574464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 215091, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5288538932800293, + "objective/train/theoretical_loss": 4.049016872082065, + "objective/train/tokens_used": 389100000, + "theoretical_loss": 4.049016872082065, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044863590772316954, + "loss": 2.8003, + "theoretical_loss": 4.049016872082065, + "tokens_seen": 368640000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044862587763289867, + "loss": 3.0327, + "theoretical_loss": 4.04893523976756, + "tokens_seen": 368705536 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486158475426279, + "loss": 2.9907, + "theoretical_loss": 4.0488536260235275, + "tokens_seen": 368771072 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004486058174523571, + "loss": 2.9698, + "theoretical_loss": 4.048772030842443, + "tokens_seen": 368836608 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044859578736208627, + "loss": 3.0078, + "theoretical_loss": 4.048690454216788, + "tokens_seen": 368902144 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044858575727181545, + "loss": 2.8499, + "theoretical_loss": 4.048608896139048, + "tokens_seen": 368967680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485757271815447, + "loss": 3.1026, + "theoretical_loss": 4.048527356601712, + "tokens_seen": 369033216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485656970912738, + "loss": 2.9261, + "theoretical_loss": 4.048445835597272, + "tokens_seen": 369098752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044855566700100304, + "loss": 2.9621, + "theoretical_loss": 4.04836433311823, + "tokens_seen": 369164288 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044854563691073217, + "loss": 2.8368, + "theoretical_loss": 4.0482828491570855, + "tokens_seen": 369229824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485356068204614, + "loss": 2.7932, + "theoretical_loss": 4.048201383706347, + "tokens_seen": 369295360 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004485255767301906, + "loss": 2.7461, + "theoretical_loss": 4.048119936758525, + "tokens_seen": 369360896 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044851554663991977, + "loss": 2.9926, + "theoretical_loss": 4.048038508306135, + "tokens_seen": 369426432 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044850551654964895, + "loss": 2.8286, + "theoretical_loss": 4.047957098341698, + "tokens_seen": 369491968 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044849548645937813, + "loss": 2.9703, + "theoretical_loss": 4.047875706857737, + "tokens_seen": 369557504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484854563691073, + "loss": 2.7894, + "theoretical_loss": 4.047794333846781, + "tokens_seen": 369623040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044847542627883655, + "loss": 2.9906, + "theoretical_loss": 4.047712979301364, + "tokens_seen": 369688576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484653961885657, + "loss": 2.8056, + "theoretical_loss": 4.047631643214023, + "tokens_seen": 369754112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484553660982949, + "loss": 2.4839, + "theoretical_loss": 4.0475503255772995, + "tokens_seen": 369819648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044844533600802404, + "loss": 2.7831, + "theoretical_loss": 4.047469026383739, + "tokens_seen": 369885184 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044843530591775327, + "loss": 2.9546, + "theoretical_loss": 4.047387745625892, + "tokens_seen": 369950720 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044842527582748245, + "loss": 2.7135, + "theoretical_loss": 4.047306483296315, + "tokens_seen": 370016256 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044841524573721163, + "loss": 2.861, + "theoretical_loss": 4.047225239387565, + "tokens_seen": 370081792 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004484052156469408, + "loss": 2.876, + "theoretical_loss": 4.047144013892206, + "tokens_seen": 370147328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044839518555667005, + "loss": 2.8395, + "theoretical_loss": 4.047062806802805, + "tokens_seen": 370212864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 215564, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0313522815704346, + "objective/train/theoretical_loss": 4.046981618111936, + "objective/train/tokens_used": 390738400, + "theoretical_loss": 4.046981618111936, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483851554663992, + "loss": 2.9527, + "theoretical_loss": 4.046981618111936, + "tokens_seen": 370278400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483751253761284, + "loss": 2.9376, + "theoretical_loss": 4.046900447812173, + "tokens_seen": 370343936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044836509528585754, + "loss": 2.9443, + "theoretical_loss": 4.046819295896098, + "tokens_seen": 370409472 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483550651955868, + "loss": 3.0019, + "theoretical_loss": 4.046738162356295, + "tokens_seen": 370475008 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044834503510531596, + "loss": 2.8873, + "theoretical_loss": 4.046657047185354, + "tokens_seen": 370540544 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044833500501504514, + "loss": 3.11, + "theoretical_loss": 4.0465759503758685, + "tokens_seen": 370606080 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483249749247743, + "loss": 2.805, + "theoretical_loss": 4.046494871920436, + "tokens_seen": 370671616 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483149448345035, + "loss": 2.853, + "theoretical_loss": 4.046413811811659, + "tokens_seen": 370737152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004483049147442327, + "loss": 3.0758, + "theoretical_loss": 4.0463327700421425, + "tokens_seen": 370802688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482948846539619, + "loss": 2.7516, + "theoretical_loss": 4.046251746604499, + "tokens_seen": 370868224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044828485456369104, + "loss": 2.9978, + "theoretical_loss": 4.046170741491343, + "tokens_seen": 370933760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482748244734203, + "loss": 2.7836, + "theoretical_loss": 4.046089754695292, + "tokens_seen": 370999296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482647943831494, + "loss": 2.7176, + "theoretical_loss": 4.046008786208972, + "tokens_seen": 371064832 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044825476429287864, + "loss": 2.9292, + "theoretical_loss": 4.04592783602501, + "tokens_seen": 371130368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482447342026078, + "loss": 2.9083, + "theoretical_loss": 4.045846904136037, + "tokens_seen": 371195904 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448234704112337, + "loss": 2.6722, + "theoretical_loss": 4.04576599053469, + "tokens_seen": 371261440 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482246740220662, + "loss": 3.207, + "theoretical_loss": 4.045685095213611, + "tokens_seen": 371326976 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004482146439317954, + "loss": 2.7582, + "theoretical_loss": 4.045604218165442, + "tokens_seen": 371392512 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044820461384152455, + "loss": 2.6807, + "theoretical_loss": 4.045523359382835, + "tokens_seen": 371458048 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481945837512538, + "loss": 3.2028, + "theoretical_loss": 4.045442518858441, + "tokens_seen": 371523584 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044818455366098296, + "loss": 2.8334, + "theoretical_loss": 4.045361696584919, + "tokens_seen": 371589120 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044817452357071214, + "loss": 2.8413, + "theoretical_loss": 4.04528089255493, + "tokens_seen": 371654656 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481644934804414, + "loss": 3.0689, + "theoretical_loss": 4.045200106761142, + "tokens_seen": 371720192 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481544633901705, + "loss": 3.1544, + "theoretical_loss": 4.045119339196222, + "tokens_seen": 371785728 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044814443329989974, + "loss": 2.6545, + "theoretical_loss": 4.045038589852847, + "tokens_seen": 371851264 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 216610, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.039722442626953, + "objective/train/theoretical_loss": 4.044957858723696, + "objective/train/tokens_used": 392376800, + "theoretical_loss": 4.044957858723696, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044813440320962887, + "loss": 2.9195, + "theoretical_loss": 4.044957858723696, + "tokens_seen": 371916800 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481243731193581, + "loss": 2.9107, + "theoretical_loss": 4.0448771458014505, + "tokens_seen": 371982336 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004481143430290873, + "loss": 2.8144, + "theoretical_loss": 4.044796451078799, + "tokens_seen": 372047872 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044810431293881647, + "loss": 2.8264, + "theoretical_loss": 4.044715774548433, + "tokens_seen": 372113408 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044809428284854565, + "loss": 2.7547, + "theoretical_loss": 4.044635116203047, + "tokens_seen": 372178944 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480842527582749, + "loss": 2.8847, + "theoretical_loss": 4.0445544760353425, + "tokens_seen": 372244480 + }, + { + "epoch": 0.11, + "learning_rate": 0.000448074222668004, + "loss": 2.7869, + "theoretical_loss": 4.044473854038022, + "tokens_seen": 372310016 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044806419257773324, + "loss": 2.9045, + "theoretical_loss": 4.044393250203795, + "tokens_seen": 372375552 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044805416248746237, + "loss": 3.1393, + "theoretical_loss": 4.044312664525374, + "tokens_seen": 372441088 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480441323971916, + "loss": 2.9725, + "theoretical_loss": 4.044232096995476, + "tokens_seen": 372506624 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004480341023069208, + "loss": 2.984, + "theoretical_loss": 4.044151547606821, + "tokens_seen": 372572160 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044802407221664997, + "loss": 2.9168, + "theoretical_loss": 4.0440710163521345, + "tokens_seen": 372637696 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044801404212637915, + "loss": 2.9977, + "theoretical_loss": 4.043990503224146, + "tokens_seen": 372703232 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044800401203610833, + "loss": 3.0303, + "theoretical_loss": 4.04391000821559, + "tokens_seen": 372768768 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479939819458375, + "loss": 3.2721, + "theoretical_loss": 4.0438295313192025, + "tokens_seen": 372834304 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044798395185556675, + "loss": 2.6246, + "theoretical_loss": 4.043749072527727, + "tokens_seen": 372899840 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479739217652959, + "loss": 2.7876, + "theoretical_loss": 4.043668631833908, + "tokens_seen": 372965376 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004479638916750251, + "loss": 2.6676, + "theoretical_loss": 4.043588209230498, + "tokens_seen": 373030912 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044795386158475424, + "loss": 2.7663, + "theoretical_loss": 4.04350780471025, + "tokens_seen": 373096448 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044794383149448347, + "loss": 2.9916, + "theoretical_loss": 4.043427418265923, + "tokens_seen": 373161984 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044793380140421265, + "loss": 2.8541, + "theoretical_loss": 4.043347049890279, + "tokens_seen": 373227520 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044792377131394183, + "loss": 2.8811, + "theoretical_loss": 4.043266699576086, + "tokens_seen": 373293056 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447913741223671, + "loss": 2.8581, + "theoretical_loss": 4.043186367316116, + "tokens_seen": 373358592 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044790371113340025, + "loss": 2.804, + "theoretical_loss": 4.043106053103141, + "tokens_seen": 373424128 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478936810431294, + "loss": 2.9816, + "theoretical_loss": 4.043025756929945, + "tokens_seen": 373489664 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 217105, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.010817766189575, + "objective/train/theoretical_loss": 4.042945478789308, + "objective/train/tokens_used": 394015200, + "theoretical_loss": 4.042945478789308, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478836509528586, + "loss": 3.1259, + "theoretical_loss": 4.042945478789308, + "tokens_seen": 373555200 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044787362086258774, + "loss": 3.0529, + "theoretical_loss": 4.042865218674019, + "tokens_seen": 373620736 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447863590772317, + "loss": 2.7388, + "theoretical_loss": 4.04278497657687, + "tokens_seen": 373686272 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044785356068204616, + "loss": 2.8478, + "theoretical_loss": 4.042704752490658, + "tokens_seen": 373751808 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044784353059177534, + "loss": 2.7947, + "theoretical_loss": 4.042624546408182, + "tokens_seen": 373817344 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478335005015045, + "loss": 2.8891, + "theoretical_loss": 4.042544358322247, + "tokens_seen": 373882880 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478234704112337, + "loss": 2.9725, + "theoretical_loss": 4.042464188225661, + "tokens_seen": 373948416 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478134403209629, + "loss": 2.9604, + "theoretical_loss": 4.042384036111236, + "tokens_seen": 374013952 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004478034102306921, + "loss": 2.8276, + "theoretical_loss": 4.04230390197179, + "tokens_seen": 374079488 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044779338014042124, + "loss": 3.1022, + "theoretical_loss": 4.042223785800144, + "tokens_seen": 374145024 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477833500501505, + "loss": 3.1569, + "theoretical_loss": 4.042143687589123, + "tokens_seen": 374210560 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477733199598796, + "loss": 2.868, + "theoretical_loss": 4.042063607331555, + "tokens_seen": 374276096 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044776328986960884, + "loss": 2.8411, + "theoretical_loss": 4.0419835450202735, + "tokens_seen": 374341632 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447753259779338, + "loss": 3.0814, + "theoretical_loss": 4.041903500648116, + "tokens_seen": 374407168 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477432296890672, + "loss": 2.9514, + "theoretical_loss": 4.041823474207925, + "tokens_seen": 374472704 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477331995987964, + "loss": 3.0362, + "theoretical_loss": 4.041743465692545, + "tokens_seen": 374538240 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004477231695085256, + "loss": 2.8953, + "theoretical_loss": 4.041663475094826, + "tokens_seen": 374603776 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044771313941825475, + "loss": 2.8501, + "theoretical_loss": 4.041583502407622, + "tokens_seen": 374669312 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447703109327984, + "loss": 3.0705, + "theoretical_loss": 4.041503547623791, + "tokens_seen": 374734848 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476930792377131, + "loss": 2.8879, + "theoretical_loss": 4.041423610736195, + "tokens_seen": 374800384 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044768304914744234, + "loss": 2.801, + "theoretical_loss": 4.0413436917376995, + "tokens_seen": 374865920 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476730190571715, + "loss": 2.9406, + "theoretical_loss": 4.041263790621175, + "tokens_seen": 374931456 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476629889669007, + "loss": 2.859, + "theoretical_loss": 4.041183907379495, + "tokens_seen": 374996992 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476529588766299, + "loss": 2.9117, + "theoretical_loss": 4.041104042005539, + "tokens_seen": 375062528 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044764292878635907, + "loss": 3.1132, + "theoretical_loss": 4.04102419449219, + "tokens_seen": 375128064 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 218363, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9804539680480957, + "objective/train/theoretical_loss": 4.040944364832332, + "objective/train/tokens_used": 395653600, + "theoretical_loss": 4.040944364832332, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044763289869608825, + "loss": 2.9225, + "theoretical_loss": 4.040944364832332, + "tokens_seen": 375193600 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476228686058175, + "loss": 2.854, + "theoretical_loss": 4.040864553018858, + "tokens_seen": 375259136 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004476128385155466, + "loss": 2.7824, + "theoretical_loss": 4.040784759044662, + "tokens_seen": 375324672 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044760280842527585, + "loss": 2.8757, + "theoretical_loss": 4.040704982902642, + "tokens_seen": 375390208 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044759277833500503, + "loss": 2.9178, + "theoretical_loss": 4.0406252245857015, + "tokens_seen": 375455744 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475827482447342, + "loss": 3.1065, + "theoretical_loss": 4.0405454840867465, + "tokens_seen": 375521280 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475727181544634, + "loss": 2.7781, + "theoretical_loss": 4.040465761398689, + "tokens_seen": 375586816 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044756268806419257, + "loss": 2.9638, + "theoretical_loss": 4.0403860565144445, + "tokens_seen": 375652352 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044755265797392175, + "loss": 3.18, + "theoretical_loss": 4.040306369426929, + "tokens_seen": 375717888 + }, + { + "epoch": 0.11, + "learning_rate": 0.000447542627883651, + "loss": 2.7766, + "theoretical_loss": 4.04022670012907, + "tokens_seen": 375783424 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475325977933801, + "loss": 2.9218, + "theoretical_loss": 4.0401470486137905, + "tokens_seen": 375848960 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044752256770310935, + "loss": 3.0006, + "theoretical_loss": 4.040067414874024, + "tokens_seen": 375914496 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475125376128385, + "loss": 2.8125, + "theoretical_loss": 4.039987798902705, + "tokens_seen": 375980032 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004475025075225677, + "loss": 3.1362, + "theoretical_loss": 4.039908200692773, + "tokens_seen": 376045568 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474924774322969, + "loss": 2.9677, + "theoretical_loss": 4.039828620237171, + "tokens_seen": 376111104 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474824473420261, + "loss": 2.9863, + "theoretical_loss": 4.039749057528846, + "tokens_seen": 376176640 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044747241725175526, + "loss": 2.858, + "theoretical_loss": 4.039669512560751, + "tokens_seen": 376242176 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044746238716148444, + "loss": 2.9564, + "theoretical_loss": 4.039589985325839, + "tokens_seen": 376307712 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474523570712136, + "loss": 2.976, + "theoretical_loss": 4.039510475817071, + "tokens_seen": 376373248 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044744232698094285, + "loss": 3.026, + "theoretical_loss": 4.03943098402741, + "tokens_seen": 376438784 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044743229689067204, + "loss": 2.7752, + "theoretical_loss": 4.039351509949824, + "tokens_seen": 376504320 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474222668004012, + "loss": 2.9442, + "theoretical_loss": 4.0392720535772835, + "tokens_seen": 376569856 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044741223671013045, + "loss": 3.1013, + "theoretical_loss": 4.039192614902763, + "tokens_seen": 376635392 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004474022066198596, + "loss": 2.8631, + "theoretical_loss": 4.0391131939192455, + "tokens_seen": 376700928 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473921765295888, + "loss": 2.9903, + "theoretical_loss": 4.039033790619711, + "tokens_seen": 376766464 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 218865, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9096364974975586, + "objective/train/theoretical_loss": 4.038954404997148, + "objective/train/tokens_used": 397292000, + "theoretical_loss": 4.038954404997148, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044738214643931794, + "loss": 3.0827, + "theoretical_loss": 4.038954404997148, + "tokens_seen": 376832000 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473721163490472, + "loss": 3.2042, + "theoretical_loss": 4.038875037044549, + "tokens_seen": 376897536 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044736208625877636, + "loss": 3.0121, + "theoretical_loss": 4.038795686754908, + "tokens_seen": 376963072 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044735205616850554, + "loss": 2.8018, + "theoretical_loss": 4.038716354121226, + "tokens_seen": 377028608 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473420260782347, + "loss": 2.963, + "theoretical_loss": 4.038637039136505, + "tokens_seen": 377094144 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473319959879639, + "loss": 2.6641, + "theoretical_loss": 4.0385577417937535, + "tokens_seen": 377159680 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473219658976931, + "loss": 3.1323, + "theoretical_loss": 4.0384784620859815, + "tokens_seen": 377225216 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004473119358074223, + "loss": 2.9769, + "theoretical_loss": 4.038399200006207, + "tokens_seen": 377290752 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044730190571715144, + "loss": 2.9038, + "theoretical_loss": 4.038319955547447, + "tokens_seen": 377356288 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472918756268807, + "loss": 2.8752, + "theoretical_loss": 4.038240728702725, + "tokens_seen": 377421824 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472818455366098, + "loss": 3.0864, + "theoretical_loss": 4.03816151946507, + "tokens_seen": 377487360 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044727181544633904, + "loss": 2.8703, + "theoretical_loss": 4.038082327827512, + "tokens_seen": 377552896 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472617853560682, + "loss": 2.896, + "theoretical_loss": 4.038003153783086, + "tokens_seen": 377618432 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472517552657974, + "loss": 3.0818, + "theoretical_loss": 4.037923997324833, + "tokens_seen": 377683968 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472417251755266, + "loss": 2.9379, + "theoretical_loss": 4.037844858445794, + "tokens_seen": 377749504 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472316950852558, + "loss": 2.9901, + "theoretical_loss": 4.037765737139018, + "tokens_seen": 377815040 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044722166499498495, + "loss": 2.9231, + "theoretical_loss": 4.037686633397555, + "tokens_seen": 377880576 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472116349047142, + "loss": 2.7279, + "theoretical_loss": 4.03760754721446, + "tokens_seen": 377946112 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004472016048144433, + "loss": 2.8789, + "theoretical_loss": 4.037528478582793, + "tokens_seen": 378011648 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044719157472417254, + "loss": 2.8981, + "theoretical_loss": 4.037449427495615, + "tokens_seen": 378077184 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471815446339017, + "loss": 3.0708, + "theoretical_loss": 4.037370393945996, + "tokens_seen": 378142720 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471715145436309, + "loss": 2.9477, + "theoretical_loss": 4.037291377927005, + "tokens_seen": 378208256 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471614844533601, + "loss": 2.8476, + "theoretical_loss": 4.0372123794317165, + "tokens_seen": 378273792 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044715145436308927, + "loss": 3.0008, + "theoretical_loss": 4.037133398453209, + "tokens_seen": 378339328 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044714142427281845, + "loss": 2.7895, + "theoretical_loss": 4.037054434984566, + "tokens_seen": 378404864 + }, + { + "epoch": 0.11, + "objective/train/docs_used": 220023, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9392900466918945, + "objective/train/theoretical_loss": 4.036975489018874, + "objective/train/tokens_used": 398930400, + "theoretical_loss": 4.036975489018874, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471313941825477, + "loss": 2.8371, + "theoretical_loss": 4.036975489018874, + "tokens_seen": 378470400 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004471213640922768, + "loss": 3.062, + "theoretical_loss": 4.036896560549224, + "tokens_seen": 378535936 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044711133400200605, + "loss": 2.847, + "theoretical_loss": 4.03681764956871, + "tokens_seen": 378601472 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044710130391173523, + "loss": 2.8412, + "theoretical_loss": 4.036738756070429, + "tokens_seen": 378667008 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470912738214644, + "loss": 2.7814, + "theoretical_loss": 4.036659880047486, + "tokens_seen": 378732544 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470812437311936, + "loss": 2.9737, + "theoretical_loss": 4.036581021492984, + "tokens_seen": 378798080 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044707121364092277, + "loss": 2.8625, + "theoretical_loss": 4.036502180400036, + "tokens_seen": 378863616 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044706118355065195, + "loss": 2.8385, + "theoretical_loss": 4.036423356761754, + "tokens_seen": 378929152 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470511534603812, + "loss": 2.8222, + "theoretical_loss": 4.036344550571257, + "tokens_seen": 378994688 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470411233701103, + "loss": 2.8922, + "theoretical_loss": 4.0362657618216655, + "tokens_seen": 379060224 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044703109327983955, + "loss": 2.5815, + "theoretical_loss": 4.036186990506107, + "tokens_seen": 379125760 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470210631895687, + "loss": 2.9568, + "theoretical_loss": 4.036108236617711, + "tokens_seen": 379191296 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470110330992979, + "loss": 2.9341, + "theoretical_loss": 4.036029500149609, + "tokens_seen": 379256832 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004470010030090271, + "loss": 2.8396, + "theoretical_loss": 4.03595078109494, + "tokens_seen": 379322368 + }, + { + "epoch": 0.11, + "learning_rate": 0.0004469909729187563, + "loss": 3.1731, + "theoretical_loss": 4.035872079446846, + "tokens_seen": 379387904 + }, + { + "epoch": 0.11, + "learning_rate": 0.00044698094282848546, + "loss": 2.8303, + "theoretical_loss": 4.03579339519847, + "tokens_seen": 379453440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044697091273821464, + "loss": 2.9496, + "theoretical_loss": 4.035714728342962, + "tokens_seen": 379518976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469608826479438, + "loss": 2.9665, + "theoretical_loss": 4.035636078873477, + "tokens_seen": 379584512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044695085255767305, + "loss": 2.8545, + "theoretical_loss": 4.035557446783168, + "tokens_seen": 379650048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469408224674022, + "loss": 3.0728, + "theoretical_loss": 4.035478832065198, + "tokens_seen": 379715584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469307923771314, + "loss": 2.7782, + "theoretical_loss": 4.035400234712732, + "tokens_seen": 379781120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469207622868606, + "loss": 2.9358, + "theoretical_loss": 4.035321654718937, + "tokens_seen": 379846656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004469107321965898, + "loss": 2.9011, + "theoretical_loss": 4.0352430920769855, + "tokens_seen": 379912192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044690070210631896, + "loss": 2.8626, + "theoretical_loss": 4.035164546780055, + "tokens_seen": 379977728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044689067201604814, + "loss": 2.9872, + "theoretical_loss": 4.035086018821325, + "tokens_seen": 380043264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 221040, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.943068742752075, + "objective/train/theoretical_loss": 4.035007508193978, + "objective/train/tokens_used": 400568800, + "theoretical_loss": 4.035007508193978, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468806419257773, + "loss": 3.0626, + "theoretical_loss": 4.035007508193978, + "tokens_seen": 380108800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044687061183550656, + "loss": 2.7622, + "theoretical_loss": 4.034929014891202, + "tokens_seen": 380174336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468605817452357, + "loss": 2.9553, + "theoretical_loss": 4.03485053890619, + "tokens_seen": 380239872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468505516549649, + "loss": 2.7883, + "theoretical_loss": 4.034772080232137, + "tokens_seen": 380305408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044684052156469405, + "loss": 2.8372, + "theoretical_loss": 4.034693638862241, + "tokens_seen": 380370944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468304914744233, + "loss": 2.9443, + "theoretical_loss": 4.034615214789707, + "tokens_seen": 380436480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044682046138415246, + "loss": 2.7825, + "theoretical_loss": 4.034536808007741, + "tokens_seen": 380502016 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044681043129388164, + "loss": 2.8113, + "theoretical_loss": 4.034458418509553, + "tokens_seen": 380567552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004468004012036108, + "loss": 2.8747, + "theoretical_loss": 4.034380046288359, + "tokens_seen": 380633088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044679037111334, + "loss": 2.9584, + "theoretical_loss": 4.034301691337377, + "tokens_seen": 380698624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467803410230692, + "loss": 2.9068, + "theoretical_loss": 4.034223353649828, + "tokens_seen": 380764160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467703109327984, + "loss": 2.9041, + "theoretical_loss": 4.034145033218941, + "tokens_seen": 380829696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044676028084252755, + "loss": 2.9763, + "theoretical_loss": 4.034066730037944, + "tokens_seen": 380895232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467502507522568, + "loss": 3.2191, + "theoretical_loss": 4.0339884441000695, + "tokens_seen": 380960768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044674022066198597, + "loss": 3.015, + "theoretical_loss": 4.0339101753985585, + "tokens_seen": 381026304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044673019057171515, + "loss": 3.0623, + "theoretical_loss": 4.033831923926649, + "tokens_seen": 381091840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044672016048144433, + "loss": 2.8022, + "theoretical_loss": 4.033753689677588, + "tokens_seen": 381157376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467101303911735, + "loss": 3.0212, + "theoretical_loss": 4.033675472644624, + "tokens_seen": 381222912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004467001003009027, + "loss": 2.8385, + "theoretical_loss": 4.03359727282101, + "tokens_seen": 381288448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466900702106319, + "loss": 2.9087, + "theoretical_loss": 4.033519090200002, + "tokens_seen": 381353984 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466800401203611, + "loss": 2.8946, + "theoretical_loss": 4.033440924774862, + "tokens_seen": 381419520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466700100300903, + "loss": 2.7513, + "theoretical_loss": 4.033362776538853, + "tokens_seen": 381485056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044665997993981947, + "loss": 2.809, + "theoretical_loss": 4.033284645485243, + "tokens_seen": 381550592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044664994984954865, + "loss": 2.8946, + "theoretical_loss": 4.033206531607304, + "tokens_seen": 381616128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004466399197592779, + "loss": 2.8353, + "theoretical_loss": 4.033128434898312, + "tokens_seen": 381681664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 221564, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5393569469451904, + "objective/train/theoretical_loss": 4.033050355351545, + "objective/train/tokens_used": 402207200, + "theoretical_loss": 4.033050355351545, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446629889669007, + "loss": 2.8139, + "theoretical_loss": 4.033050355351545, + "tokens_seen": 381747200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044661985957873625, + "loss": 2.8216, + "theoretical_loss": 4.032972292960288, + "tokens_seen": 381812736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044660982948846543, + "loss": 2.8422, + "theoretical_loss": 4.0328942477178265, + "tokens_seen": 381878272 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465997993981946, + "loss": 2.9652, + "theoretical_loss": 4.032816219617452, + "tokens_seen": 381943808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465897693079238, + "loss": 2.8776, + "theoretical_loss": 4.032738208652459, + "tokens_seen": 382009344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044657973921765297, + "loss": 2.8477, + "theoretical_loss": 4.032660214816145, + "tokens_seen": 382074880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044656970912738215, + "loss": 2.8979, + "theoretical_loss": 4.0325822381018135, + "tokens_seen": 382140416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465596790371114, + "loss": 2.9445, + "theoretical_loss": 4.03250427850277, + "tokens_seen": 382205952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465496489468405, + "loss": 3.0591, + "theoretical_loss": 4.032426336012321, + "tokens_seen": 382271488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044653961885656975, + "loss": 2.8794, + "theoretical_loss": 4.032348410623785, + "tokens_seen": 382337024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465295887662989, + "loss": 3.0788, + "theoretical_loss": 4.032270502330475, + "tokens_seen": 382402560 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465195586760281, + "loss": 2.9419, + "theoretical_loss": 4.032192611125714, + "tokens_seen": 382468096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004465095285857573, + "loss": 3.056, + "theoretical_loss": 4.032114737002825, + "tokens_seen": 382533632 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464994984954865, + "loss": 3.0452, + "theoretical_loss": 4.032036879955138, + "tokens_seen": 382599168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044648946840521566, + "loss": 2.9026, + "theoretical_loss": 4.031959039975983, + "tokens_seen": 382664704 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044647943831494484, + "loss": 2.8125, + "theoretical_loss": 4.031881217058698, + "tokens_seen": 382730240 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446469408224674, + "loss": 2.6746, + "theoretical_loss": 4.0318034111966226, + "tokens_seen": 382795776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044645937813440325, + "loss": 2.946, + "theoretical_loss": 4.031725622383099, + "tokens_seen": 382861312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464493480441324, + "loss": 3.0812, + "theoretical_loss": 4.031647850611473, + "tokens_seen": 382926848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464393179538616, + "loss": 3.153, + "theoretical_loss": 4.031570095875098, + "tokens_seen": 382992384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004464292878635908, + "loss": 3.0693, + "theoretical_loss": 4.031492358167329, + "tokens_seen": 383057920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044641925777332, + "loss": 3.2132, + "theoretical_loss": 4.031414637481521, + "tokens_seen": 383123456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044640922768304916, + "loss": 2.6272, + "theoretical_loss": 4.031336933811038, + "tokens_seen": 383188992 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044639919759277834, + "loss": 3.0864, + "theoretical_loss": 4.031259247149245, + "tokens_seen": 383254528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463891675025075, + "loss": 3.0017, + "theoretical_loss": 4.031181577489512, + "tokens_seen": 383320064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 222655, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.689690589904785, + "objective/train/theoretical_loss": 4.031103924825214, + "objective/train/tokens_used": 403845600, + "theoretical_loss": 4.031103924825214, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044637913741223676, + "loss": 3.0431, + "theoretical_loss": 4.031103924825214, + "tokens_seen": 383385600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463691073219659, + "loss": 2.8683, + "theoretical_loss": 4.0310262891497235, + "tokens_seen": 383451136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463590772316951, + "loss": 3.0141, + "theoretical_loss": 4.030948670456425, + "tokens_seen": 383516672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044634904714142425, + "loss": 2.9239, + "theoretical_loss": 4.0308710687386995, + "tokens_seen": 383582208 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004463390170511535, + "loss": 2.7962, + "theoretical_loss": 4.030793483989939, + "tokens_seen": 383647744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044632898696088266, + "loss": 2.9929, + "theoretical_loss": 4.0307159162035315, + "tokens_seen": 383713280 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044631895687061184, + "loss": 2.9842, + "theoretical_loss": 4.030638365372874, + "tokens_seen": 383778816 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446308926780341, + "loss": 2.8692, + "theoretical_loss": 4.030560831491365, + "tokens_seen": 383844352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462988966900702, + "loss": 3.0569, + "theoretical_loss": 4.030483314552408, + "tokens_seen": 383909888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462888665997994, + "loss": 3.2353, + "theoretical_loss": 4.03040581454941, + "tokens_seen": 383975424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462788365095286, + "loss": 3.0781, + "theoretical_loss": 4.03032833147578, + "tokens_seen": 384040960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044626880641925775, + "loss": 2.7578, + "theoretical_loss": 4.030250865324932, + "tokens_seen": 384106496 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446258776328987, + "loss": 3.0714, + "theoretical_loss": 4.030173416090284, + "tokens_seen": 384172032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044624874623871617, + "loss": 2.9921, + "theoretical_loss": 4.030095983765257, + "tokens_seen": 384237568 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044623871614844535, + "loss": 2.9743, + "theoretical_loss": 4.030018568343276, + "tokens_seen": 384303104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044622868605817453, + "loss": 3.0737, + "theoretical_loss": 4.02994116981777, + "tokens_seen": 384368640 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462186559679037, + "loss": 2.7114, + "theoretical_loss": 4.02986378818217, + "tokens_seen": 384434176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004462086258776329, + "loss": 2.8272, + "theoretical_loss": 4.029786423429913, + "tokens_seen": 384499712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461985957873621, + "loss": 3.1081, + "theoretical_loss": 4.029709075554439, + "tokens_seen": 384565248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044618856569709125, + "loss": 2.938, + "theoretical_loss": 4.02963174454919, + "tokens_seen": 384630784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461785356068205, + "loss": 2.6041, + "theoretical_loss": 4.029554430407613, + "tokens_seen": 384696320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461685055165496, + "loss": 2.8779, + "theoretical_loss": 4.0294771331231605, + "tokens_seen": 384761856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044615847542627885, + "loss": 2.8709, + "theoretical_loss": 4.029399852689284, + "tokens_seen": 384827392 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044614844533600803, + "loss": 3.0417, + "theoretical_loss": 4.0293225890994435, + "tokens_seen": 384892928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461384152457372, + "loss": 2.4958, + "theoretical_loss": 4.0292453423471, + "tokens_seen": 384958464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 222910, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9024524688720703, + "objective/train/theoretical_loss": 4.029168112425719, + "objective/train/tokens_used": 405484000, + "theoretical_loss": 4.029168112425719, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004461283851554664, + "loss": 3.0308, + "theoretical_loss": 4.029168112425719, + "tokens_seen": 385024000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044611835506519563, + "loss": 2.8817, + "theoretical_loss": 4.029090899328769, + "tokens_seen": 385089536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044610832497492476, + "loss": 2.8394, + "theoretical_loss": 4.029013703049722, + "tokens_seen": 385155072 + }, + { + "epoch": 0.12, + "learning_rate": 0.000446098294884654, + "loss": 2.9092, + "theoretical_loss": 4.028936523582055, + "tokens_seen": 385220608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460882647943831, + "loss": 2.9076, + "theoretical_loss": 4.028859360919247, + "tokens_seen": 385286144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044607823470411235, + "loss": 3.1037, + "theoretical_loss": 4.0287822150547825, + "tokens_seen": 385351680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044606820461384153, + "loss": 2.9683, + "theoretical_loss": 4.028705085982148, + "tokens_seen": 385417216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460581745235707, + "loss": 2.8876, + "theoretical_loss": 4.028627973694833, + "tokens_seen": 385482752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460481444332999, + "loss": 2.7867, + "theoretical_loss": 4.028550878186334, + "tokens_seen": 385548288 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460381143430291, + "loss": 3.0148, + "theoretical_loss": 4.028473799450149, + "tokens_seen": 385613824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044602808425275826, + "loss": 2.4792, + "theoretical_loss": 4.028396737479777, + "tokens_seen": 385679360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460180541624875, + "loss": 3.0135, + "theoretical_loss": 4.028319692268725, + "tokens_seen": 385744896 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004460080240722166, + "loss": 3.1208, + "theoretical_loss": 4.028242663810501, + "tokens_seen": 385810432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044599799398194586, + "loss": 2.9895, + "theoretical_loss": 4.028165652098618, + "tokens_seen": 385875968 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445987963891675, + "loss": 3.0448, + "theoretical_loss": 4.028088657126593, + "tokens_seen": 385941504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459779338014042, + "loss": 3.0796, + "theoretical_loss": 4.028011678887944, + "tokens_seen": 386007040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459679037111334, + "loss": 3.0512, + "theoretical_loss": 4.027934717376195, + "tokens_seen": 386072576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459578736208626, + "loss": 2.7118, + "theoretical_loss": 4.027857772584873, + "tokens_seen": 386138112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044594784353059176, + "loss": 3.0313, + "theoretical_loss": 4.027780844507508, + "tokens_seen": 386203648 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445937813440321, + "loss": 2.834, + "theoretical_loss": 4.0277039331376345, + "tokens_seen": 386269184 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004459277833500502, + "loss": 2.7997, + "theoretical_loss": 4.027627038468791, + "tokens_seen": 386334720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044591775325977936, + "loss": 2.9824, + "theoretical_loss": 4.027550160494516, + "tokens_seen": 386400256 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044590772316950854, + "loss": 3.1209, + "theoretical_loss": 4.027473299208357, + "tokens_seen": 386465792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458976930792377, + "loss": 3.1398, + "theoretical_loss": 4.027396454603862, + "tokens_seen": 386531328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044588766298896696, + "loss": 2.7702, + "theoretical_loss": 4.027319626674583, + "tokens_seen": 386596864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 224330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.956303596496582, + "objective/train/theoretical_loss": 4.027242815414074, + "objective/train/tokens_used": 407122400, + "theoretical_loss": 4.027242815414074, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458776328986961, + "loss": 3.0091, + "theoretical_loss": 4.027242815414074, + "tokens_seen": 386662400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458676028084253, + "loss": 3.0036, + "theoretical_loss": 4.027166020815896, + "tokens_seen": 386727936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044585757271815445, + "loss": 2.829, + "theoretical_loss": 4.027089242873612, + "tokens_seen": 386793472 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458475426278837, + "loss": 3.0081, + "theoretical_loss": 4.027012481580787, + "tokens_seen": 386859008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044583751253761286, + "loss": 2.9494, + "theoretical_loss": 4.026935736930992, + "tokens_seen": 386924544 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044582748244734204, + "loss": 2.8553, + "theoretical_loss": 4.0268590089178, + "tokens_seen": 386990080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458174523570712, + "loss": 2.8579, + "theoretical_loss": 4.026782297534787, + "tokens_seen": 387055616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004458074222668004, + "loss": 2.9371, + "theoretical_loss": 4.026705602775536, + "tokens_seen": 387121152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457973921765296, + "loss": 2.8851, + "theoretical_loss": 4.02662892463363, + "tokens_seen": 387186688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457873620862588, + "loss": 3.1423, + "theoretical_loss": 4.026552263102657, + "tokens_seen": 387252224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044577733199598795, + "loss": 2.9309, + "theoretical_loss": 4.026475618176208, + "tokens_seen": 387317760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457673019057172, + "loss": 3.0442, + "theoretical_loss": 4.026398989847877, + "tokens_seen": 387383296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044575727181544637, + "loss": 3.0007, + "theoretical_loss": 4.0263223781112645, + "tokens_seen": 387448832 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044574724172517555, + "loss": 3.0223, + "theoretical_loss": 4.026245782959971, + "tokens_seen": 387514368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044573721163490473, + "loss": 2.7477, + "theoretical_loss": 4.026169204387603, + "tokens_seen": 387579904 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457271815446339, + "loss": 2.8504, + "theoretical_loss": 4.026092642387769, + "tokens_seen": 387645440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457171514543631, + "loss": 2.9407, + "theoretical_loss": 4.0260160969540815, + "tokens_seen": 387710976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004457071213640923, + "loss": 3.1619, + "theoretical_loss": 4.025939568080156, + "tokens_seen": 387776512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044569709127382145, + "loss": 2.8439, + "theoretical_loss": 4.025863055759615, + "tokens_seen": 387842048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456870611835507, + "loss": 2.932, + "theoretical_loss": 4.025786559986079, + "tokens_seen": 387907584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456770310932798, + "loss": 2.8177, + "theoretical_loss": 4.025710080753175, + "tokens_seen": 387973120 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044566700100300905, + "loss": 2.9714, + "theoretical_loss": 4.025633618054535, + "tokens_seen": 388038656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044565697091273823, + "loss": 3.0176, + "theoretical_loss": 4.025557171883791, + "tokens_seen": 388104192 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456469408224674, + "loss": 3.023, + "theoretical_loss": 4.025480742234581, + "tokens_seen": 388169728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456369107321966, + "loss": 2.8862, + "theoretical_loss": 4.025404329100546, + "tokens_seen": 388235264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 225094, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.5328869819641113, + "objective/train/theoretical_loss": 4.0253279324753315, + "objective/train/tokens_used": 408760800, + "theoretical_loss": 4.0253279324753315, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044562688064192583, + "loss": 3.2148, + "theoretical_loss": 4.0253279324753315, + "tokens_seen": 388300800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044561685055165496, + "loss": 2.9028, + "theoretical_loss": 4.025251552352584, + "tokens_seen": 388366336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004456068204613842, + "loss": 3.0211, + "theoretical_loss": 4.025175188725955, + "tokens_seen": 388431872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455967903711133, + "loss": 2.894, + "theoretical_loss": 4.025098841589099, + "tokens_seen": 388497408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044558676028084255, + "loss": 2.7945, + "theoretical_loss": 4.025022510935676, + "tokens_seen": 388562944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044557673019057173, + "loss": 2.944, + "theoretical_loss": 4.024946196759347, + "tokens_seen": 388628480 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455667001003009, + "loss": 3.0782, + "theoretical_loss": 4.024869899053778, + "tokens_seen": 388694016 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455566700100301, + "loss": 3.1571, + "theoretical_loss": 4.024793617812637, + "tokens_seen": 388759552 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455466399197593, + "loss": 2.8881, + "theoretical_loss": 4.024717353029597, + "tokens_seen": 388825088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044553660982948846, + "loss": 2.9853, + "theoretical_loss": 4.024641104698335, + "tokens_seen": 388890624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455265797392177, + "loss": 2.8279, + "theoretical_loss": 4.02456487281253, + "tokens_seen": 388956160 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004455165496489468, + "loss": 2.8868, + "theoretical_loss": 4.024488657365863, + "tokens_seen": 389021696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044550651955867606, + "loss": 2.901, + "theoretical_loss": 4.024412458352023, + "tokens_seen": 389087232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454964894684052, + "loss": 3.0881, + "theoretical_loss": 4.024336275764698, + "tokens_seen": 389152768 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454864593781344, + "loss": 2.9645, + "theoretical_loss": 4.024260109597583, + "tokens_seen": 389218304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454764292878636, + "loss": 2.8748, + "theoretical_loss": 4.024183959844375, + "tokens_seen": 389283840 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454663991975928, + "loss": 2.8245, + "theoretical_loss": 4.024107826498772, + "tokens_seen": 389349376 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044545636910732196, + "loss": 2.966, + "theoretical_loss": 4.024031709554482, + "tokens_seen": 389414912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454463390170512, + "loss": 2.7041, + "theoretical_loss": 4.023955609005208, + "tokens_seen": 389480448 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454363089267803, + "loss": 2.8466, + "theoretical_loss": 4.023879524844663, + "tokens_seen": 389545984 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044542627883650956, + "loss": 3.0461, + "theoretical_loss": 4.023803457066561, + "tokens_seen": 389611520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454162487462387, + "loss": 3.013, + "theoretical_loss": 4.023727405664619, + "tokens_seen": 389677056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004454062186559679, + "loss": 3.1216, + "theoretical_loss": 4.02365137063256, + "tokens_seen": 389742592 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453961885656971, + "loss": 2.8899, + "theoretical_loss": 4.023575351964108, + "tokens_seen": 389808128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453861584754263, + "loss": 2.9788, + "theoretical_loss": 4.02349934965299, + "tokens_seen": 389873664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 226325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.121893882751465, + "objective/train/theoretical_loss": 4.023423363692939, + "objective/train/tokens_used": 410399200, + "theoretical_loss": 4.023423363692939, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044537612838515547, + "loss": 2.988, + "theoretical_loss": 4.023423363692939, + "tokens_seen": 389939200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044536609829488465, + "loss": 2.8528, + "theoretical_loss": 4.023347394077689, + "tokens_seen": 390004736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044535606820461383, + "loss": 3.0006, + "theoretical_loss": 4.023271440800979, + "tokens_seen": 390070272 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044534603811434306, + "loss": 2.818, + "theoretical_loss": 4.023195503856551, + "tokens_seen": 390135808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453360080240722, + "loss": 2.8456, + "theoretical_loss": 4.02311958323815, + "tokens_seen": 390201344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453259779338014, + "loss": 3.0276, + "theoretical_loss": 4.023043678939525, + "tokens_seen": 390266880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044531594784353055, + "loss": 2.9954, + "theoretical_loss": 4.022967790954429, + "tokens_seen": 390332416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004453059177532598, + "loss": 3.021, + "theoretical_loss": 4.022891919276617, + "tokens_seen": 390397952 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044529588766298897, + "loss": 2.8376, + "theoretical_loss": 4.022816063899848, + "tokens_seen": 390463488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044528585757271815, + "loss": 2.9268, + "theoretical_loss": 4.022740224817886, + "tokens_seen": 390529024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044527582748244733, + "loss": 2.9293, + "theoretical_loss": 4.022664402024495, + "tokens_seen": 390594560 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044526579739217657, + "loss": 3.2044, + "theoretical_loss": 4.0225885955134455, + "tokens_seen": 390660096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452557673019057, + "loss": 3.0436, + "theoretical_loss": 4.022512805278511, + "tokens_seen": 390725632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044524573721163493, + "loss": 2.7897, + "theoretical_loss": 4.022437031313467, + "tokens_seen": 390791168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044523570712136406, + "loss": 3.1187, + "theoretical_loss": 4.022361273612093, + "tokens_seen": 390856704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004452256770310933, + "loss": 3.0885, + "theoretical_loss": 4.022285532168174, + "tokens_seen": 390922240 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044521564694082247, + "loss": 3.0223, + "theoretical_loss": 4.022209806975494, + "tokens_seen": 390987776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044520561685055165, + "loss": 2.9709, + "theoretical_loss": 4.022134098027844, + "tokens_seen": 391053312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044519558676028083, + "loss": 2.9389, + "theoretical_loss": 4.022058405319018, + "tokens_seen": 391118848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044518555667001, + "loss": 2.7071, + "theoretical_loss": 4.021982728842812, + "tokens_seen": 391184384 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044517552657973925, + "loss": 2.8463, + "theoretical_loss": 4.021907068593027, + "tokens_seen": 391249920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044516549648946843, + "loss": 2.8327, + "theoretical_loss": 4.021831424563467, + "tokens_seen": 391315456 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451554663991976, + "loss": 2.7659, + "theoretical_loss": 4.021755796747938, + "tokens_seen": 391380992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451454363089268, + "loss": 2.8349, + "theoretical_loss": 4.02168018514025, + "tokens_seen": 391446528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044513540621865603, + "loss": 2.9836, + "theoretical_loss": 4.021604589734219, + "tokens_seen": 391512064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 226938, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0995724201202393, + "objective/train/theoretical_loss": 4.021529010523659, + "objective/train/tokens_used": 412037600, + "theoretical_loss": 4.021529010523659, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044512537612838516, + "loss": 3.0031, + "theoretical_loss": 4.021529010523659, + "tokens_seen": 391577600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451153460381144, + "loss": 2.8716, + "theoretical_loss": 4.0214534475023935, + "tokens_seen": 391643136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004451053159478435, + "loss": 2.902, + "theoretical_loss": 4.0213779006642465, + "tokens_seen": 391708672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044509528585757275, + "loss": 2.9234, + "theoretical_loss": 4.021302370003044, + "tokens_seen": 391774208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044508525576730194, + "loss": 2.9735, + "theoretical_loss": 4.021226855512616, + "tokens_seen": 391839744 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450752256770311, + "loss": 2.805, + "theoretical_loss": 4.021151357186799, + "tokens_seen": 391905280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450651955867603, + "loss": 2.8537, + "theoretical_loss": 4.021075875019429, + "tokens_seen": 391970816 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450551654964895, + "loss": 2.779, + "theoretical_loss": 4.021000409004348, + "tokens_seen": 392036352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044504513540621866, + "loss": 2.9278, + "theoretical_loss": 4.0209249591353995, + "tokens_seen": 392101888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450351053159479, + "loss": 2.9617, + "theoretical_loss": 4.020849525406431, + "tokens_seen": 392167424 + }, + { + "epoch": 0.12, + "learning_rate": 0.000445025075225677, + "loss": 2.8302, + "theoretical_loss": 4.020774107811294, + "tokens_seen": 392232960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044501504513540626, + "loss": 2.8883, + "theoretical_loss": 4.020698706343844, + "tokens_seen": 392298496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004450050150451354, + "loss": 2.8845, + "theoretical_loss": 4.020623320997937, + "tokens_seen": 392364032 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449949849548646, + "loss": 2.9504, + "theoretical_loss": 4.020547951767435, + "tokens_seen": 392429568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449849548645938, + "loss": 2.6795, + "theoretical_loss": 4.020472598646203, + "tokens_seen": 392495104 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444974924774323, + "loss": 2.8708, + "theoretical_loss": 4.020397261628108, + "tokens_seen": 392560640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044496489468405216, + "loss": 2.8069, + "theoretical_loss": 4.020321940707021, + "tokens_seen": 392626176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449548645937814, + "loss": 2.8997, + "theoretical_loss": 4.020246635876817, + "tokens_seen": 392691712 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449448345035105, + "loss": 2.9409, + "theoretical_loss": 4.020171347131376, + "tokens_seen": 392757248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044493480441323976, + "loss": 2.8469, + "theoretical_loss": 4.0200960744645755, + "tokens_seen": 392822784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449247743229689, + "loss": 2.9276, + "theoretical_loss": 4.020020817870303, + "tokens_seen": 392888320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449147442326981, + "loss": 3.0255, + "theoretical_loss": 4.019945577342445, + "tokens_seen": 392953856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004449047141424273, + "loss": 2.949, + "theoretical_loss": 4.019870352874894, + "tokens_seen": 393019392 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448946840521565, + "loss": 2.9662, + "theoretical_loss": 4.019795144461543, + "tokens_seen": 393084928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044488465396188567, + "loss": 2.9866, + "theoretical_loss": 4.019719952096294, + "tokens_seen": 393150464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 228118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.819366931915283, + "objective/train/theoretical_loss": 4.019644775773043, + "objective/train/tokens_used": 413676000, + "theoretical_loss": 4.019644775773043, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044487462387161485, + "loss": 2.8016, + "theoretical_loss": 4.019644775773043, + "tokens_seen": 393216000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044486459378134403, + "loss": 3.015, + "theoretical_loss": 4.019569615485698, + "tokens_seen": 393281536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044485456369107326, + "loss": 2.7138, + "theoretical_loss": 4.019494471228166, + "tokens_seen": 393347072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448445336008024, + "loss": 3.0278, + "theoretical_loss": 4.019419342994358, + "tokens_seen": 393412608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004448345035105316, + "loss": 2.8665, + "theoretical_loss": 4.01934423077819, + "tokens_seen": 393478144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044482447342026075, + "loss": 3.0124, + "theoretical_loss": 4.019269134573579, + "tokens_seen": 393543680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044481444332999, + "loss": 2.9158, + "theoretical_loss": 4.019194054374447, + "tokens_seen": 393609216 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044480441323971917, + "loss": 3.0232, + "theoretical_loss": 4.019118990174718, + "tokens_seen": 393674752 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044479438314944835, + "loss": 2.8318, + "theoretical_loss": 4.019043941968319, + "tokens_seen": 393740288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044478435305917753, + "loss": 2.8996, + "theoretical_loss": 4.018968909749184, + "tokens_seen": 393805824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044477432296890677, + "loss": 2.8596, + "theoretical_loss": 4.018893893511246, + "tokens_seen": 393871360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447642928786359, + "loss": 3.0738, + "theoretical_loss": 4.018818893248442, + "tokens_seen": 393936896 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044475426278836513, + "loss": 2.8301, + "theoretical_loss": 4.018743908954715, + "tokens_seen": 394002432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044474423269809426, + "loss": 2.8219, + "theoretical_loss": 4.018668940624007, + "tokens_seen": 394067968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004447342026078235, + "loss": 2.9127, + "theoretical_loss": 4.018593988250269, + "tokens_seen": 394133504 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044472417251755267, + "loss": 2.9686, + "theoretical_loss": 4.01851905182745, + "tokens_seen": 394199040 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044471414242728185, + "loss": 2.9248, + "theoretical_loss": 4.018444131349505, + "tokens_seen": 394264576 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044470411233701103, + "loss": 2.8442, + "theoretical_loss": 4.018369226810392, + "tokens_seen": 394330112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446940822467402, + "loss": 2.9457, + "theoretical_loss": 4.018294338204071, + "tokens_seen": 394395648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446840521564694, + "loss": 2.7568, + "theoretical_loss": 4.018219465524508, + "tokens_seen": 394461184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044467402206619863, + "loss": 2.8902, + "theoretical_loss": 4.0181446087656685, + "tokens_seen": 394526720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044466399197592776, + "loss": 2.8016, + "theoretical_loss": 4.018069767921524, + "tokens_seen": 394592256 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444653961885657, + "loss": 3.0541, + "theoretical_loss": 4.01799494298605, + "tokens_seen": 394657792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446439317953861, + "loss": 2.636, + "theoretical_loss": 4.017920133953223, + "tokens_seen": 394723328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044463390170511536, + "loss": 2.9731, + "theoretical_loss": 4.017845340817024, + "tokens_seen": 394788864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 228787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5924181938171387, + "objective/train/theoretical_loss": 4.017770563571435, + "objective/train/tokens_used": 415314400, + "theoretical_loss": 4.017770563571435, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044462387161484454, + "loss": 2.844, + "theoretical_loss": 4.017770563571435, + "tokens_seen": 394854400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446138415245737, + "loss": 2.8228, + "theoretical_loss": 4.017695802210447, + "tokens_seen": 394919936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004446038114343029, + "loss": 2.9139, + "theoretical_loss": 4.017621056728047, + "tokens_seen": 394985472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044459378134403214, + "loss": 3.0017, + "theoretical_loss": 4.017546327118231, + "tokens_seen": 395051008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044458375125376126, + "loss": 2.9105, + "theoretical_loss": 4.017471613374996, + "tokens_seen": 395116544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445737211634905, + "loss": 2.7957, + "theoretical_loss": 4.01739691549234, + "tokens_seen": 395182080 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445636910732196, + "loss": 2.9505, + "theoretical_loss": 4.017322233464269, + "tokens_seen": 395247616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044455366098294886, + "loss": 2.8694, + "theoretical_loss": 4.017247567284789, + "tokens_seen": 395313152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044454363089267804, + "loss": 2.6045, + "theoretical_loss": 4.017172916947911, + "tokens_seen": 395378688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445336008024072, + "loss": 2.9287, + "theoretical_loss": 4.017098282447646, + "tokens_seen": 395444224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445235707121364, + "loss": 2.9332, + "theoretical_loss": 4.017023663778014, + "tokens_seen": 395509760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004445135406218656, + "loss": 2.7736, + "theoretical_loss": 4.016949060933033, + "tokens_seen": 395575296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044450351053159477, + "loss": 2.9836, + "theoretical_loss": 4.016874473906726, + "tokens_seen": 395640832 + }, + { + "epoch": 0.12, + "learning_rate": 0.000444493480441324, + "loss": 2.7708, + "theoretical_loss": 4.016799902693119, + "tokens_seen": 395706368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044448345035105313, + "loss": 2.9282, + "theoretical_loss": 4.016725347286243, + "tokens_seen": 395771904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044447342026078236, + "loss": 2.9519, + "theoretical_loss": 4.01665080768013, + "tokens_seen": 395837440 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444633901705115, + "loss": 2.6818, + "theoretical_loss": 4.016576283868816, + "tokens_seen": 395902976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444533600802407, + "loss": 2.7103, + "theoretical_loss": 4.016501775846342, + "tokens_seen": 395968512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044444332998996996, + "loss": 2.9564, + "theoretical_loss": 4.016427283606747, + "tokens_seen": 396034048 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444332998996991, + "loss": 2.9737, + "theoretical_loss": 4.01635280714408, + "tokens_seen": 396099584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444232698094283, + "loss": 2.8439, + "theoretical_loss": 4.016278346452389, + "tokens_seen": 396165120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444132397191575, + "loss": 2.7579, + "theoretical_loss": 4.016203901525726, + "tokens_seen": 396230656 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004444032096288867, + "loss": 2.9436, + "theoretical_loss": 4.0161294723581475, + "tokens_seen": 396296192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044439317953861587, + "loss": 2.9978, + "theoretical_loss": 4.016055058943712, + "tokens_seen": 396361728 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044438314944834505, + "loss": 2.9448, + "theoretical_loss": 4.01598066127648, + "tokens_seen": 396427264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 229924, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7313523292541504, + "objective/train/theoretical_loss": 4.015906279350517, + "objective/train/tokens_used": 416952800, + "theoretical_loss": 4.015906279350517, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044437311935807423, + "loss": 2.895, + "theoretical_loss": 4.015906279350517, + "tokens_seen": 396492800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044436308926780346, + "loss": 2.8399, + "theoretical_loss": 4.015831913159893, + "tokens_seen": 396558336 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443530591775326, + "loss": 3.0352, + "theoretical_loss": 4.015757562698678, + "tokens_seen": 396623872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443430290872618, + "loss": 2.971, + "theoretical_loss": 4.015683227960948, + "tokens_seen": 396689408 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044433299899699095, + "loss": 2.7352, + "theoretical_loss": 4.015608908940781, + "tokens_seen": 396754944 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004443229689067202, + "loss": 2.7234, + "theoretical_loss": 4.015534605632256, + "tokens_seen": 396820480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044431293881644937, + "loss": 2.9999, + "theoretical_loss": 4.01546031802946, + "tokens_seen": 396886016 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044430290872617855, + "loss": 3.0871, + "theoretical_loss": 4.01538604612648, + "tokens_seen": 396951552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044429287863590773, + "loss": 2.8029, + "theoretical_loss": 4.015311789917406, + "tokens_seen": 397017088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044428284854563697, + "loss": 2.9118, + "theoretical_loss": 4.015237549396333, + "tokens_seen": 397082624 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442728184553661, + "loss": 2.869, + "theoretical_loss": 4.015163324557358, + "tokens_seen": 397148160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044426278836509533, + "loss": 3.1997, + "theoretical_loss": 4.015089115394581, + "tokens_seen": 397213696 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044425275827482446, + "loss": 2.8647, + "theoretical_loss": 4.015014921902106, + "tokens_seen": 397279232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442427281845537, + "loss": 2.787, + "theoretical_loss": 4.014940744074039, + "tokens_seen": 397344768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044423269809428287, + "loss": 2.9288, + "theoretical_loss": 4.014866581904491, + "tokens_seen": 397410304 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044422266800401205, + "loss": 2.8941, + "theoretical_loss": 4.014792435387575, + "tokens_seen": 397475840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044421263791374123, + "loss": 3.0604, + "theoretical_loss": 4.014718304517408, + "tokens_seen": 397541376 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004442026078234704, + "loss": 2.9054, + "theoretical_loss": 4.0146441892881075, + "tokens_seen": 397606912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441925777331996, + "loss": 3.0049, + "theoretical_loss": 4.014570089693798, + "tokens_seen": 397672448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044418254764292883, + "loss": 3.119, + "theoretical_loss": 4.014496005728605, + "tokens_seen": 397737984 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044417251755265796, + "loss": 2.8411, + "theoretical_loss": 4.014421937386658, + "tokens_seen": 397803520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441624874623872, + "loss": 3.0238, + "theoretical_loss": 4.014347884662088, + "tokens_seen": 397869056 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441524573721163, + "loss": 2.7703, + "theoretical_loss": 4.014273847549032, + "tokens_seen": 397934592 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044414242728184556, + "loss": 2.7537, + "theoretical_loss": 4.014199826041628, + "tokens_seen": 398000128 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044413239719157474, + "loss": 2.898, + "theoretical_loss": 4.014125820134016, + "tokens_seen": 398065664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 231183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1082088947296143, + "objective/train/theoretical_loss": 4.014051829820343, + "objective/train/tokens_used": 418591200, + "theoretical_loss": 4.014051829820343, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441223671013039, + "loss": 3.0483, + "theoretical_loss": 4.014051829820343, + "tokens_seen": 398131200 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004441123370110331, + "loss": 2.9731, + "theoretical_loss": 4.0139778550947565, + "tokens_seen": 398196736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044410230692076234, + "loss": 2.8605, + "theoretical_loss": 4.013903895951408, + "tokens_seen": 398262272 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044409227683049146, + "loss": 2.9152, + "theoretical_loss": 4.013829952384452, + "tokens_seen": 398327808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440822467402207, + "loss": 3.0129, + "theoretical_loss": 4.013756024388045, + "tokens_seen": 398393344 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440722166499498, + "loss": 2.9997, + "theoretical_loss": 4.013682111956348, + "tokens_seen": 398458880 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044406218655967906, + "loss": 2.7039, + "theoretical_loss": 4.013608215083526, + "tokens_seen": 398524416 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044405215646940824, + "loss": 2.8974, + "theoretical_loss": 4.0135343337637455, + "tokens_seen": 398589952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440421263791374, + "loss": 2.7087, + "theoretical_loss": 4.013460467991176, + "tokens_seen": 398655488 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440320962888666, + "loss": 2.8162, + "theoretical_loss": 4.01338661775999, + "tokens_seen": 398721024 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440220661985958, + "loss": 3.114, + "theoretical_loss": 4.013312783064366, + "tokens_seen": 398786560 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044401203610832497, + "loss": 2.8509, + "theoretical_loss": 4.013238963898483, + "tokens_seen": 398852096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004440020060180542, + "loss": 2.6743, + "theoretical_loss": 4.0131651602565235, + "tokens_seen": 398917632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044399197592778333, + "loss": 2.7401, + "theoretical_loss": 4.013091372132672, + "tokens_seen": 398983168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044398194583751256, + "loss": 3.0041, + "theoretical_loss": 4.01301759952112, + "tokens_seen": 399048704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439719157472417, + "loss": 2.8825, + "theoretical_loss": 4.012943842416057, + "tokens_seen": 399114240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439618856569709, + "loss": 2.8614, + "theoretical_loss": 4.01287010081168, + "tokens_seen": 399179776 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439518555667001, + "loss": 2.7441, + "theoretical_loss": 4.012796374702187, + "tokens_seen": 399245312 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439418254764293, + "loss": 2.7377, + "theoretical_loss": 4.012722664081779, + "tokens_seen": 399310848 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044393179538615847, + "loss": 2.8877, + "theoretical_loss": 4.012648968944661, + "tokens_seen": 399376384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004439217652958877, + "loss": 2.9744, + "theoretical_loss": 4.01257528928504, + "tokens_seen": 399441920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044391173520561683, + "loss": 2.6387, + "theoretical_loss": 4.012501625097128, + "tokens_seen": 399507456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044390170511534607, + "loss": 2.8492, + "theoretical_loss": 4.012427976375139, + "tokens_seen": 399572992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438916750250752, + "loss": 2.9283, + "theoretical_loss": 4.012354343113288, + "tokens_seen": 399638528 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044388164493480443, + "loss": 2.8775, + "theoretical_loss": 4.012280725305798, + "tokens_seen": 399704064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 231706, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.14528226852417, + "objective/train/theoretical_loss": 4.01220712294689, + "objective/train/tokens_used": 420229600, + "theoretical_loss": 4.01220712294689, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438716148445336, + "loss": 3.0579, + "theoretical_loss": 4.01220712294689, + "tokens_seen": 399769600 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438615847542628, + "loss": 2.7976, + "theoretical_loss": 4.012133536030791, + "tokens_seen": 399835136 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044385155466399197, + "loss": 2.9258, + "theoretical_loss": 4.012059964551731, + "tokens_seen": 399900672 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044384152457372115, + "loss": 2.7009, + "theoretical_loss": 4.011986408503942, + "tokens_seen": 399966208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044383149448345033, + "loss": 2.7834, + "theoretical_loss": 4.0119128678816605, + "tokens_seen": 400031744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044382146439317957, + "loss": 2.9803, + "theoretical_loss": 4.011839342679125, + "tokens_seen": 400097280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004438114343029087, + "loss": 2.6878, + "theoretical_loss": 4.0117658328905765, + "tokens_seen": 400162816 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044380140421263793, + "loss": 2.8165, + "theoretical_loss": 4.011692338510262, + "tokens_seen": 400228352 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437913741223671, + "loss": 2.6641, + "theoretical_loss": 4.011618859532426, + "tokens_seen": 400293888 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437813440320963, + "loss": 2.8163, + "theoretical_loss": 4.011545395951323, + "tokens_seen": 400359424 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437713139418255, + "loss": 2.6949, + "theoretical_loss": 4.011471947761207, + "tokens_seen": 400424960 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044376128385155466, + "loss": 3.0059, + "theoretical_loss": 4.011398514956333, + "tokens_seen": 400490496 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044375125376128384, + "loss": 2.8145, + "theoretical_loss": 4.011325097530963, + "tokens_seen": 400556032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044374122367101307, + "loss": 2.6344, + "theoretical_loss": 4.011251695479361, + "tokens_seen": 400621568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437311935807422, + "loss": 3.0702, + "theoretical_loss": 4.011178308795792, + "tokens_seen": 400687104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044372116349047143, + "loss": 2.8061, + "theoretical_loss": 4.011104937474527, + "tokens_seen": 400752640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044371113340020056, + "loss": 2.6934, + "theoretical_loss": 4.011031581509838, + "tokens_seen": 400818176 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004437011033099298, + "loss": 2.8448, + "theoretical_loss": 4.0109582408960005, + "tokens_seen": 400883712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044369107321965903, + "loss": 3.0339, + "theoretical_loss": 4.010884915627294, + "tokens_seen": 400949248 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044368104312938816, + "loss": 2.77, + "theoretical_loss": 4.0108116056980005, + "tokens_seen": 401014784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436710130391174, + "loss": 2.9177, + "theoretical_loss": 4.010738311102405, + "tokens_seen": 401080320 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436609829488465, + "loss": 2.8799, + "theoretical_loss": 4.010665031834796, + "tokens_seen": 401145856 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044365095285857576, + "loss": 2.6816, + "theoretical_loss": 4.010591767889462, + "tokens_seen": 401211392 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044364092276830494, + "loss": 2.627, + "theoretical_loss": 4.0105185192607, + "tokens_seen": 401276928 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436308926780341, + "loss": 2.8835, + "theoretical_loss": 4.010445285942806, + "tokens_seen": 401342464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 232929, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.049757242202759, + "objective/train/theoretical_loss": 4.0103720679300805, + "objective/train/tokens_used": 421868000, + "theoretical_loss": 4.0103720679300805, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004436208625877633, + "loss": 2.743, + "theoretical_loss": 4.0103720679300805, + "tokens_seen": 401408000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044361083249749254, + "loss": 2.7237, + "theoretical_loss": 4.010298865216827, + "tokens_seen": 401473536 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044360080240722166, + "loss": 2.8752, + "theoretical_loss": 4.010225677797351, + "tokens_seen": 401539072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435907723169509, + "loss": 2.9571, + "theoretical_loss": 4.010152505665964, + "tokens_seen": 401604608 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044358074222668, + "loss": 2.823, + "theoretical_loss": 4.010079348816976, + "tokens_seen": 401670144 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044357071213640926, + "loss": 2.8259, + "theoretical_loss": 4.010006207244704, + "tokens_seen": 401735680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044356068204613844, + "loss": 2.7683, + "theoretical_loss": 4.009933080943466, + "tokens_seen": 401801216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435506519558676, + "loss": 3.0118, + "theoretical_loss": 4.009859969907584, + "tokens_seen": 401866752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435406218655968, + "loss": 2.8564, + "theoretical_loss": 4.0097868741313825, + "tokens_seen": 401932288 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443530591775326, + "loss": 2.7656, + "theoretical_loss": 4.009713793609189, + "tokens_seen": 401997824 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044352056168505517, + "loss": 2.6552, + "theoretical_loss": 4.009640728335333, + "tokens_seen": 402063360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004435105315947844, + "loss": 2.995, + "theoretical_loss": 4.009567678304151, + "tokens_seen": 402128896 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044350050150451353, + "loss": 2.8946, + "theoretical_loss": 4.009494643509978, + "tokens_seen": 402194432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044349047141424276, + "loss": 2.6877, + "theoretical_loss": 4.009421623947153, + "tokens_seen": 402259968 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434804413239719, + "loss": 2.9107, + "theoretical_loss": 4.00934861961002, + "tokens_seen": 402325504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434704112337011, + "loss": 3.044, + "theoretical_loss": 4.009275630492926, + "tokens_seen": 402391040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434603811434303, + "loss": 2.9364, + "theoretical_loss": 4.009202656590217, + "tokens_seen": 402456576 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434503510531595, + "loss": 2.9931, + "theoretical_loss": 4.009129697896247, + "tokens_seen": 402522112 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044344032096288867, + "loss": 2.7369, + "theoretical_loss": 4.00905675440537, + "tokens_seen": 402587648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434302908726179, + "loss": 2.7316, + "theoretical_loss": 4.008983826111945, + "tokens_seen": 402653184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044342026078234703, + "loss": 2.8793, + "theoretical_loss": 4.008910913010331, + "tokens_seen": 402718720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044341023069207627, + "loss": 2.7036, + "theoretical_loss": 4.008838015094893, + "tokens_seen": 402784256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004434002006018054, + "loss": 2.7359, + "theoretical_loss": 4.008765132359999, + "tokens_seen": 402849792 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044339017051153463, + "loss": 2.8239, + "theoretical_loss": 4.0086922648000165, + "tokens_seen": 402915328 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433801404212638, + "loss": 3.0154, + "theoretical_loss": 4.00861941240932, + "tokens_seen": 402980864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 233431, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7376959323883057, + "objective/train/theoretical_loss": 4.008546575182286, + "objective/train/tokens_used": 423506400, + "theoretical_loss": 4.008546575182286, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443370110330993, + "loss": 2.7459, + "theoretical_loss": 4.008546575182286, + "tokens_seen": 403046400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044336008024072217, + "loss": 2.8226, + "theoretical_loss": 4.008473753113292, + "tokens_seen": 403111936 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044335005015045135, + "loss": 2.8344, + "theoretical_loss": 4.008400946196721, + "tokens_seen": 403177472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044334002006018053, + "loss": 3.1402, + "theoretical_loss": 4.008328154426956, + "tokens_seen": 403243008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044332998996990977, + "loss": 2.8666, + "theoretical_loss": 4.008255377798388, + "tokens_seen": 403308544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004433199598796389, + "loss": 2.8195, + "theoretical_loss": 4.008182616305406, + "tokens_seen": 403374080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044330992978936813, + "loss": 2.707, + "theoretical_loss": 4.008109869942404, + "tokens_seen": 403439616 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432998996990973, + "loss": 2.7543, + "theoretical_loss": 4.008037138703779, + "tokens_seen": 403505152 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432898696088265, + "loss": 3.1115, + "theoretical_loss": 4.007964422583931, + "tokens_seen": 403570688 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432798395185557, + "loss": 2.8065, + "theoretical_loss": 4.007891721577263, + "tokens_seen": 403636224 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044326980942828486, + "loss": 2.8883, + "theoretical_loss": 4.00781903567818, + "tokens_seen": 403701760 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044325977933801404, + "loss": 2.7961, + "theoretical_loss": 4.0077463648810925, + "tokens_seen": 403767296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044324974924774327, + "loss": 3.0498, + "theoretical_loss": 4.007673709180411, + "tokens_seen": 403832832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004432397191574724, + "loss": 2.8936, + "theoretical_loss": 4.007601068570549, + "tokens_seen": 403898368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044322968906720163, + "loss": 2.7508, + "theoretical_loss": 4.007528443045928, + "tokens_seen": 403963904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044321965897693076, + "loss": 3.0503, + "theoretical_loss": 4.007455832600965, + "tokens_seen": 404029440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044320962888666, + "loss": 3.0522, + "theoretical_loss": 4.007383237230086, + "tokens_seen": 404094976 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431995987963892, + "loss": 2.5872, + "theoretical_loss": 4.007310656927716, + "tokens_seen": 404160512 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044318956870611836, + "loss": 2.891, + "theoretical_loss": 4.0072380916882855, + "tokens_seen": 404226048 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044317953861584754, + "loss": 3.0283, + "theoretical_loss": 4.007165541506227, + "tokens_seen": 404291584 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431695085255767, + "loss": 2.9584, + "theoretical_loss": 4.007093006375976, + "tokens_seen": 404357120 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431594784353059, + "loss": 2.9348, + "theoretical_loss": 4.007020486291972, + "tokens_seen": 404422656 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044314944834503514, + "loss": 2.949, + "theoretical_loss": 4.006947981248654, + "tokens_seen": 404488192 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044313941825476427, + "loss": 2.7082, + "theoretical_loss": 4.006875491240468, + "tokens_seen": 404553728 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431293881644935, + "loss": 2.7423, + "theoretical_loss": 4.006803016261861, + "tokens_seen": 404619264 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 234514, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.404675245285034, + "objective/train/theoretical_loss": 4.006730556307284, + "objective/train/tokens_used": 425144800, + "theoretical_loss": 4.006730556307284, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004431193580742227, + "loss": 3.1039, + "theoretical_loss": 4.006730556307284, + "tokens_seen": 404684800 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044310932798395186, + "loss": 2.6791, + "theoretical_loss": 4.006658111371189, + "tokens_seen": 404750336 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044309929789368104, + "loss": 2.6862, + "theoretical_loss": 4.006585681448033, + "tokens_seen": 404815872 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430892678034102, + "loss": 2.8161, + "theoretical_loss": 4.006513266532275, + "tokens_seen": 404881408 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004430792377131394, + "loss": 2.8454, + "theoretical_loss": 4.006440866618378, + "tokens_seen": 404946944 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044306920762286864, + "loss": 2.9207, + "theoretical_loss": 4.006368481700806, + "tokens_seen": 405012480 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044305917753259777, + "loss": 2.7304, + "theoretical_loss": 4.0062961117740254, + "tokens_seen": 405078016 + }, + { + "epoch": 0.12, + "learning_rate": 0.000443049147442327, + "loss": 2.8102, + "theoretical_loss": 4.00622375683251, + "tokens_seen": 405143552 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044303911735205613, + "loss": 2.7443, + "theoretical_loss": 4.006151416870731, + "tokens_seen": 405209088 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044302908726178537, + "loss": 2.8585, + "theoretical_loss": 4.0060790918831675, + "tokens_seen": 405274624 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044301905717151455, + "loss": 2.832, + "theoretical_loss": 4.006006781864298, + "tokens_seen": 405340160 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044300902708124373, + "loss": 2.8704, + "theoretical_loss": 4.005934486808605, + "tokens_seen": 405405696 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429989969909729, + "loss": 2.8721, + "theoretical_loss": 4.0058622067105745, + "tokens_seen": 405471232 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429889669007021, + "loss": 3.0217, + "theoretical_loss": 4.005789941564695, + "tokens_seen": 405536768 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044297893681043127, + "loss": 2.781, + "theoretical_loss": 4.005717691365457, + "tokens_seen": 405602304 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429689067201605, + "loss": 2.7174, + "theoretical_loss": 4.005645456107356, + "tokens_seen": 405667840 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044295887662988963, + "loss": 2.5529, + "theoretical_loss": 4.005573235784888, + "tokens_seen": 405733376 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044294884653961887, + "loss": 2.9012, + "theoretical_loss": 4.005501030392553, + "tokens_seen": 405798912 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429388164493481, + "loss": 3.0316, + "theoretical_loss": 4.005428839924855, + "tokens_seen": 405864448 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044292878635907723, + "loss": 2.766, + "theoretical_loss": 4.0053566643763014, + "tokens_seen": 405929984 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044291875626880647, + "loss": 2.9476, + "theoretical_loss": 4.0052845037413976, + "tokens_seen": 405995520 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004429087261785356, + "loss": 3.0683, + "theoretical_loss": 4.005212358014658, + "tokens_seen": 406061056 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044289869608826483, + "loss": 2.8003, + "theoretical_loss": 4.005140227190596, + "tokens_seen": 406126592 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442888665997994, + "loss": 2.7452, + "theoretical_loss": 4.00506811126373, + "tokens_seen": 406192128 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428786359077232, + "loss": 2.8949, + "theoretical_loss": 4.00499601022858, + "tokens_seen": 406257664 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 235119, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8222172260284424, + "objective/train/theoretical_loss": 4.0049239240796695, + "objective/train/tokens_used": 426783200, + "theoretical_loss": 4.0049239240796695, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044286860581745237, + "loss": 2.999, + "theoretical_loss": 4.0049239240796695, + "tokens_seen": 406323200 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044285857572718155, + "loss": 2.8565, + "theoretical_loss": 4.004851852811523, + "tokens_seen": 406388736 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044284854563691073, + "loss": 2.8309, + "theoretical_loss": 4.004779796418673, + "tokens_seen": 406454272 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044283851554663997, + "loss": 2.713, + "theoretical_loss": 4.00470775489565, + "tokens_seen": 406519808 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428284854563691, + "loss": 2.9481, + "theoretical_loss": 4.004635728236988, + "tokens_seen": 406585344 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044281845536609833, + "loss": 2.8521, + "theoretical_loss": 4.004563716437227, + "tokens_seen": 406650880 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004428084252758275, + "loss": 2.8142, + "theoretical_loss": 4.004491719490905, + "tokens_seen": 406716416 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427983951855567, + "loss": 2.6657, + "theoretical_loss": 4.004419737392568, + "tokens_seen": 406781952 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427883650952859, + "loss": 2.8134, + "theoretical_loss": 4.004347770136761, + "tokens_seen": 406847488 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044277833500501506, + "loss": 2.6711, + "theoretical_loss": 4.0042758177180335, + "tokens_seen": 406913024 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044276830491474424, + "loss": 2.8276, + "theoretical_loss": 4.004203880130939, + "tokens_seen": 406978560 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044275827482447347, + "loss": 2.881, + "theoretical_loss": 4.004131957370031, + "tokens_seen": 407044096 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427482447342026, + "loss": 2.9221, + "theoretical_loss": 4.004060049429869, + "tokens_seen": 407109632 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044273821464393184, + "loss": 2.7697, + "theoretical_loss": 4.003988156305013, + "tokens_seen": 407175168 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044272818455366096, + "loss": 2.764, + "theoretical_loss": 4.003916277990028, + "tokens_seen": 407240704 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427181544633902, + "loss": 2.8386, + "theoretical_loss": 4.003844414479477, + "tokens_seen": 407306240 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004427081243731194, + "loss": 2.9933, + "theoretical_loss": 4.003772565767934, + "tokens_seen": 407371776 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044269809428284856, + "loss": 2.7289, + "theoretical_loss": 4.003700731849969, + "tokens_seen": 407437312 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044268806419257774, + "loss": 2.9191, + "theoretical_loss": 4.003628912720157, + "tokens_seen": 407502848 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426780341023069, + "loss": 2.6767, + "theoretical_loss": 4.003557108373077, + "tokens_seen": 407568384 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426680040120361, + "loss": 2.907, + "theoretical_loss": 4.00348531880331, + "tokens_seen": 407633920 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044265797392176534, + "loss": 2.7716, + "theoretical_loss": 4.003413544005439, + "tokens_seen": 407699456 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044264794383149447, + "loss": 2.8966, + "theoretical_loss": 4.003341783974051, + "tokens_seen": 407764992 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426379137412237, + "loss": 2.8671, + "theoretical_loss": 4.003270038703736, + "tokens_seen": 407830528 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004426278836509529, + "loss": 2.6621, + "theoretical_loss": 4.003198308189086, + "tokens_seen": 407896064 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 236101, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.368303060531616, + "objective/train/theoretical_loss": 4.003126592424696, + "objective/train/tokens_used": 428421600, + "theoretical_loss": 4.003126592424696, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044261785356068206, + "loss": 2.6108, + "theoretical_loss": 4.003126592424696, + "tokens_seen": 407961600 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044260782347041124, + "loss": 2.7856, + "theoretical_loss": 4.003054891405164, + "tokens_seen": 408027136 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425977933801404, + "loss": 2.846, + "theoretical_loss": 4.002983205125091, + "tokens_seen": 408092672 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425877632898696, + "loss": 2.5993, + "theoretical_loss": 4.00291153357908, + "tokens_seen": 408158208 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044257773319959884, + "loss": 2.9444, + "theoretical_loss": 4.002839876761739, + "tokens_seen": 408223744 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044256770310932797, + "loss": 2.8485, + "theoretical_loss": 4.002768234667677, + "tokens_seen": 408289280 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425576730190572, + "loss": 2.812, + "theoretical_loss": 4.002696607291506, + "tokens_seen": 408354816 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044254764292878633, + "loss": 2.9355, + "theoretical_loss": 4.0026249946278405, + "tokens_seen": 408420352 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044253761283851557, + "loss": 2.7608, + "theoretical_loss": 4.0025533966713, + "tokens_seen": 408485888 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044252758274824475, + "loss": 2.7582, + "theoretical_loss": 4.002481813416504, + "tokens_seen": 408551424 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044251755265797393, + "loss": 2.8645, + "theoretical_loss": 4.002410244858076, + "tokens_seen": 408616960 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004425075225677031, + "loss": 2.9369, + "theoretical_loss": 4.002338690990644, + "tokens_seen": 408682496 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424974924774323, + "loss": 2.7085, + "theoretical_loss": 4.0022671518088355, + "tokens_seen": 408748032 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044248746238716147, + "loss": 2.7926, + "theoretical_loss": 4.002195627307283, + "tokens_seen": 408813568 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424774322968907, + "loss": 2.7285, + "theoretical_loss": 4.002124117480621, + "tokens_seen": 408879104 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044246740220661983, + "loss": 2.8286, + "theoretical_loss": 4.002052622323489, + "tokens_seen": 408944640 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044245737211634907, + "loss": 3.0013, + "theoretical_loss": 4.001981141830525, + "tokens_seen": 409010176 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044244734202607825, + "loss": 2.5402, + "theoretical_loss": 4.001909675996375, + "tokens_seen": 409075712 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044243731193580743, + "loss": 2.7802, + "theoretical_loss": 4.001838224815683, + "tokens_seen": 409141248 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424272818455366, + "loss": 2.6874, + "theoretical_loss": 4.001766788283098, + "tokens_seen": 409206784 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004424172517552658, + "loss": 2.8716, + "theoretical_loss": 4.001695366393275, + "tokens_seen": 409272320 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442407221664995, + "loss": 2.636, + "theoretical_loss": 4.001623959140864, + "tokens_seen": 409337856 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423971915747242, + "loss": 3.065, + "theoretical_loss": 4.001552566520525, + "tokens_seen": 409403392 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044238716148445334, + "loss": 2.9232, + "theoretical_loss": 4.001481188526918, + "tokens_seen": 409468928 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044237713139418257, + "loss": 2.7951, + "theoretical_loss": 4.001409825154705, + "tokens_seen": 409534464 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 237324, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1303606033325195, + "objective/train/theoretical_loss": 4.001338476398553, + "objective/train/tokens_used": 430060000, + "theoretical_loss": 4.001338476398553, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423671013039117, + "loss": 3.0079, + "theoretical_loss": 4.001338476398553, + "tokens_seen": 409600000 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044235707121364093, + "loss": 2.9637, + "theoretical_loss": 4.0012671422531305, + "tokens_seen": 409665536 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423470411233701, + "loss": 2.6292, + "theoretical_loss": 4.001195822713108, + "tokens_seen": 409731072 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423370110330993, + "loss": 2.883, + "theoretical_loss": 4.001124517773161, + "tokens_seen": 409796608 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423269809428285, + "loss": 2.99, + "theoretical_loss": 4.001053227427966, + "tokens_seen": 409862144 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004423169508525577, + "loss": 2.7892, + "theoretical_loss": 4.000981951672202, + "tokens_seen": 409927680 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044230692076228684, + "loss": 2.9052, + "theoretical_loss": 4.000910690500552, + "tokens_seen": 409993216 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422968906720161, + "loss": 3.0222, + "theoretical_loss": 4.000839443907702, + "tokens_seen": 410058752 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422868605817452, + "loss": 2.8375, + "theoretical_loss": 4.000768211888339, + "tokens_seen": 410124288 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044227683049147444, + "loss": 2.6957, + "theoretical_loss": 4.000696994437154, + "tokens_seen": 410189824 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422668004012036, + "loss": 2.7352, + "theoretical_loss": 4.000625791548843, + "tokens_seen": 410255360 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422567703109328, + "loss": 2.7709, + "theoretical_loss": 4.0005546032181, + "tokens_seen": 410320896 + }, + { + "epoch": 0.12, + "learning_rate": 0.000442246740220662, + "loss": 2.9533, + "theoretical_loss": 4.000483429439624, + "tokens_seen": 410386432 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044223671013039116, + "loss": 2.874, + "theoretical_loss": 4.00041227020812, + "tokens_seen": 410451968 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044222668004012034, + "loss": 3.1148, + "theoretical_loss": 4.00034112551829, + "tokens_seen": 410517504 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422166499498496, + "loss": 3.0062, + "theoretical_loss": 4.000269995364842, + "tokens_seen": 410583040 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004422066198595787, + "loss": 2.9177, + "theoretical_loss": 4.000198879742488, + "tokens_seen": 410648576 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044219658976930794, + "loss": 2.7881, + "theoretical_loss": 4.000127778645939, + "tokens_seen": 410714112 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421865596790371, + "loss": 2.7334, + "theoretical_loss": 4.000056692069911, + "tokens_seen": 410779648 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421765295887663, + "loss": 2.8035, + "theoretical_loss": 3.9999856200091255, + "tokens_seen": 410845184 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044216649949849554, + "loss": 2.8879, + "theoretical_loss": 3.9999145624583017, + "tokens_seen": 410910720 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044215646940822467, + "loss": 2.8508, + "theoretical_loss": 3.9998435194121633, + "tokens_seen": 410976256 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421464393179539, + "loss": 2.8841, + "theoretical_loss": 3.999772490865439, + "tokens_seen": 411041792 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421364092276831, + "loss": 2.7457, + "theoretical_loss": 3.9997014768128567, + "tokens_seen": 411107328 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044212637913741226, + "loss": 3.0086, + "theoretical_loss": 3.9996304772491498, + "tokens_seen": 411172864 + }, + { + "epoch": 0.12, + "objective/train/docs_used": 237892, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8793866634368896, + "objective/train/theoretical_loss": 3.9995594921690536, + "objective/train/tokens_used": 431698400, + "theoretical_loss": 3.9995594921690536, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044211634904714144, + "loss": 2.8024, + "theoretical_loss": 3.9995594921690536, + "tokens_seen": 411238400 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004421063189568706, + "loss": 2.7261, + "theoretical_loss": 3.9994885215673057, + "tokens_seen": 411303936 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420962888665998, + "loss": 2.9451, + "theoretical_loss": 3.9994175654386472, + "tokens_seen": 411369472 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044208625877632904, + "loss": 2.7401, + "theoretical_loss": 3.9993466237778215, + "tokens_seen": 411435008 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044207622868605817, + "loss": 2.9415, + "theoretical_loss": 3.9992756965795744, + "tokens_seen": 411500544 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420661985957874, + "loss": 2.8241, + "theoretical_loss": 3.999204783838655, + "tokens_seen": 411566080 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044205616850551653, + "loss": 2.9024, + "theoretical_loss": 3.9991338855498153, + "tokens_seen": 411631616 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044204613841524577, + "loss": 2.9414, + "theoretical_loss": 3.9990630017078095, + "tokens_seen": 411697152 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044203610832497495, + "loss": 2.8594, + "theoretical_loss": 3.9989921323073947, + "tokens_seen": 411762688 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044202607823470413, + "loss": 2.9609, + "theoretical_loss": 3.998921277343331, + "tokens_seen": 411828224 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420160481444333, + "loss": 3.0881, + "theoretical_loss": 3.9988504368103817, + "tokens_seen": 411893760 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004420060180541625, + "loss": 2.573, + "theoretical_loss": 3.9987796107033104, + "tokens_seen": 411959296 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044199598796389167, + "loss": 2.9766, + "theoretical_loss": 3.9987087990168866, + "tokens_seen": 412024832 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419859578736209, + "loss": 2.9167, + "theoretical_loss": 3.998638001745881, + "tokens_seen": 412090368 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044197592778335003, + "loss": 2.6952, + "theoretical_loss": 3.998567218885067, + "tokens_seen": 412155904 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044196589769307927, + "loss": 2.8777, + "theoretical_loss": 3.9984964504292204, + "tokens_seen": 412221440 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044195586760280845, + "loss": 2.8639, + "theoretical_loss": 3.998425696373121, + "tokens_seen": 412286976 + }, + { + "epoch": 0.12, + "learning_rate": 0.00044194583751253763, + "loss": 2.8543, + "theoretical_loss": 3.998354956711551, + "tokens_seen": 412352512 + }, + { + "epoch": 0.12, + "learning_rate": 0.0004419358074222668, + "loss": 3.0178, + "theoretical_loss": 3.9982842314392935, + "tokens_seen": 412418048 + }, + { + "epoch": 0.12, + "learning_rate": 0.000441925777331996, + "loss": 2.7787, + "theoretical_loss": 3.9982135205511358, + "tokens_seen": 412483584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004419157472417252, + "loss": 3.1016, + "theoretical_loss": 3.9981428240418686, + "tokens_seen": 412549120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004419057171514544, + "loss": 2.6455, + "theoretical_loss": 3.998072141906284, + "tokens_seen": 412614656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044189568706118354, + "loss": 2.9787, + "theoretical_loss": 3.998001474139178, + "tokens_seen": 412680192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044188565697091277, + "loss": 2.9904, + "theoretical_loss": 3.997930820735348, + "tokens_seen": 412745728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418756268806419, + "loss": 2.9195, + "theoretical_loss": 3.9978601816895942, + "tokens_seen": 412811264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 239244, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.011247396469116, + "objective/train/theoretical_loss": 3.9977895569967217, + "objective/train/tokens_used": 433336800, + "theoretical_loss": 3.9977895569967217, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044186559679037113, + "loss": 2.8852, + "theoretical_loss": 3.9977895569967217, + "tokens_seen": 412876800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418555667001003, + "loss": 2.8744, + "theoretical_loss": 3.997718946651535, + "tokens_seen": 412942336 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418455366098295, + "loss": 2.5262, + "theoretical_loss": 3.997648350648843, + "tokens_seen": 413007872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418355065195587, + "loss": 2.8229, + "theoretical_loss": 3.997577768983459, + "tokens_seen": 413073408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418254764292879, + "loss": 2.7307, + "theoretical_loss": 3.997507201650196, + "tokens_seen": 413138944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044181544633901704, + "loss": 2.87, + "theoretical_loss": 3.9974366486438706, + "tokens_seen": 413204480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004418054162487463, + "loss": 2.7706, + "theoretical_loss": 3.9973661099593034, + "tokens_seen": 413270016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417953861584754, + "loss": 2.9375, + "theoretical_loss": 3.9972955855913157, + "tokens_seen": 413335552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044178535606820464, + "loss": 2.6413, + "theoretical_loss": 3.9972250755347334, + "tokens_seen": 413401088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417753259779338, + "loss": 2.741, + "theoretical_loss": 3.997154579784384, + "tokens_seen": 413466624 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441765295887663, + "loss": 2.8083, + "theoretical_loss": 3.997084098335098, + "tokens_seen": 413532160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417552657973922, + "loss": 2.9568, + "theoretical_loss": 3.997013631181708, + "tokens_seen": 413597696 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044174523570712136, + "loss": 2.906, + "theoretical_loss": 3.99694317831905, + "tokens_seen": 413663232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044173520561685054, + "loss": 2.9089, + "theoretical_loss": 3.9968727397419634, + "tokens_seen": 413728768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417251755265798, + "loss": 3.0527, + "theoretical_loss": 3.9968023154452883, + "tokens_seen": 413794304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004417151454363089, + "loss": 2.7741, + "theoretical_loss": 3.9967319054238684, + "tokens_seen": 413859840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044170511534603814, + "loss": 2.9615, + "theoretical_loss": 3.996661509672551, + "tokens_seen": 413925376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044169508525576727, + "loss": 3.0461, + "theoretical_loss": 3.996591128186185, + "tokens_seen": 413990912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416850551654965, + "loss": 2.776, + "theoretical_loss": 3.996520760959622, + "tokens_seen": 414056448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416750250752257, + "loss": 2.8094, + "theoretical_loss": 3.996450407987716, + "tokens_seen": 414121984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044166499498495487, + "loss": 2.9947, + "theoretical_loss": 3.9963800692653253, + "tokens_seen": 414187520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044165496489468405, + "loss": 2.7043, + "theoretical_loss": 3.9963097447873097, + "tokens_seen": 414253056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416449348044133, + "loss": 2.8336, + "theoretical_loss": 3.9962394345485315, + "tokens_seen": 414318592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004416349047141424, + "loss": 2.6923, + "theoretical_loss": 3.9961691385438556, + "tokens_seen": 414384128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044162487462387164, + "loss": 2.8808, + "theoretical_loss": 3.99609885676815, + "tokens_seen": 414449664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 239946, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0474255084991455, + "objective/train/theoretical_loss": 3.9960285892162855, + "objective/train/tokens_used": 434975200, + "theoretical_loss": 3.9960285892162855, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044161484453360077, + "loss": 2.9599, + "theoretical_loss": 3.9960285892162855, + "tokens_seen": 414515200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044160481444333, + "loss": 2.5837, + "theoretical_loss": 3.995958335883135, + "tokens_seen": 414580736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415947843530592, + "loss": 2.6826, + "theoretical_loss": 3.995888096763574, + "tokens_seen": 414646272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044158475426278837, + "loss": 2.7732, + "theoretical_loss": 3.9958178718524824, + "tokens_seen": 414711808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044157472417251755, + "loss": 2.8478, + "theoretical_loss": 3.9957476611447396, + "tokens_seen": 414777344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044156469408224673, + "loss": 2.7315, + "theoretical_loss": 3.9956774646352304, + "tokens_seen": 414842880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415546639919759, + "loss": 2.796, + "theoretical_loss": 3.9956072823188418, + "tokens_seen": 414908416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044154463390170515, + "loss": 2.9685, + "theoretical_loss": 3.9955371141904616, + "tokens_seen": 414973952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415346038114343, + "loss": 2.9153, + "theoretical_loss": 3.995466960244982, + "tokens_seen": 415039488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004415245737211635, + "loss": 2.7571, + "theoretical_loss": 3.995396820477298, + "tokens_seen": 415105024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044151454363089264, + "loss": 2.8539, + "theoretical_loss": 3.995326694882306, + "tokens_seen": 415170560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044150451354062187, + "loss": 2.9172, + "theoretical_loss": 3.9952565834549056, + "tokens_seen": 415236096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044149448345035105, + "loss": 2.9305, + "theoretical_loss": 3.9951864861899997, + "tokens_seen": 415301632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044148445336008023, + "loss": 2.9859, + "theoretical_loss": 3.995116403082493, + "tokens_seen": 415367168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414744232698094, + "loss": 2.755, + "theoretical_loss": 3.995046334127293, + "tokens_seen": 415432704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044146439317953865, + "loss": 2.809, + "theoretical_loss": 3.99497627931931, + "tokens_seen": 415498240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414543630892678, + "loss": 2.9747, + "theoretical_loss": 3.994906238653457, + "tokens_seen": 415563776 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441444332998997, + "loss": 2.8364, + "theoretical_loss": 3.9948362121246497, + "tokens_seen": 415629312 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414343029087262, + "loss": 2.613, + "theoretical_loss": 3.9947661997278052, + "tokens_seen": 415694848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414242728184554, + "loss": 2.5021, + "theoretical_loss": 3.9946962014578453, + "tokens_seen": 415760384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004414142427281846, + "loss": 2.8959, + "theoretical_loss": 3.994626217309693, + "tokens_seen": 415825920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044140421263791374, + "loss": 2.8188, + "theoretical_loss": 3.994556247278275, + "tokens_seen": 415891456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044139418254764297, + "loss": 2.9287, + "theoretical_loss": 3.994486291358519, + "tokens_seen": 415956992 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413841524573721, + "loss": 2.9442, + "theoretical_loss": 3.9944163495453564, + "tokens_seen": 416022528 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044137412236710133, + "loss": 2.8413, + "theoretical_loss": 3.994346421833721, + "tokens_seen": 416088064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 241045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6522488594055176, + "objective/train/theoretical_loss": 3.99427650821855, + "objective/train/tokens_used": 436613600, + "theoretical_loss": 3.99427650821855, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413640922768305, + "loss": 2.9212, + "theoretical_loss": 3.99427650821855, + "tokens_seen": 416153600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413540621865597, + "loss": 2.8946, + "theoretical_loss": 3.994206608694782, + "tokens_seen": 416219136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413440320962889, + "loss": 2.7445, + "theoretical_loss": 3.994136723257358, + "tokens_seen": 416284672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413340020060181, + "loss": 2.782, + "theoretical_loss": 3.994066851901224, + "tokens_seen": 416350208 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044132397191574724, + "loss": 2.8814, + "theoretical_loss": 3.993996994621326, + "tokens_seen": 416415744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413139418254765, + "loss": 2.82, + "theoretical_loss": 3.9939271514126125, + "tokens_seen": 416481280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004413039117352056, + "loss": 2.7661, + "theoretical_loss": 3.9938573222700375, + "tokens_seen": 416546816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044129388164493484, + "loss": 2.9603, + "theoretical_loss": 3.9937875071885554, + "tokens_seen": 416612352 + }, + { + "epoch": 0.13, + "learning_rate": 0.000441283851554664, + "loss": 3.1666, + "theoretical_loss": 3.993717706163122, + "tokens_seen": 416677888 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412738214643932, + "loss": 2.768, + "theoretical_loss": 3.9936479191886995, + "tokens_seen": 416743424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412637913741224, + "loss": 2.9516, + "theoretical_loss": 3.9935781462602487, + "tokens_seen": 416808960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044125376128385156, + "loss": 2.82, + "theoretical_loss": 3.9935083873727355, + "tokens_seen": 416874496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044124373119358074, + "loss": 2.9864, + "theoretical_loss": 3.9934386425211277, + "tokens_seen": 416940032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044123370110331, + "loss": 2.8075, + "theoretical_loss": 3.9933689117003954, + "tokens_seen": 417005568 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004412236710130391, + "loss": 2.7716, + "theoretical_loss": 3.9932991949055117, + "tokens_seen": 417071104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044121364092276834, + "loss": 2.6797, + "theoretical_loss": 3.993229492131452, + "tokens_seen": 417136640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044120361083249747, + "loss": 2.8761, + "theoretical_loss": 3.9931598033731945, + "tokens_seen": 417202176 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411935807422267, + "loss": 2.9552, + "theoretical_loss": 3.99309012862572, + "tokens_seen": 417267712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411835506519559, + "loss": 2.7448, + "theoretical_loss": 3.993020467884011, + "tokens_seen": 417333248 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044117352056168507, + "loss": 2.6875, + "theoretical_loss": 3.9929508211430553, + "tokens_seen": 417398784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044116349047141425, + "loss": 2.7963, + "theoretical_loss": 3.9928811883978392, + "tokens_seen": 417464320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411534603811435, + "loss": 2.988, + "theoretical_loss": 3.9928115696433553, + "tokens_seen": 417529856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411434302908726, + "loss": 2.705, + "theoretical_loss": 3.992741964874596, + "tokens_seen": 417595392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044113340020060184, + "loss": 2.7923, + "theoretical_loss": 3.9926723740865584, + "tokens_seen": 417660928 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044112337011033097, + "loss": 2.6621, + "theoretical_loss": 3.992602797274241, + "tokens_seen": 417726464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 241540, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0460522174835205, + "objective/train/theoretical_loss": 3.9925332344326447, + "objective/train/tokens_used": 438252000, + "theoretical_loss": 3.9925332344326447, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411133400200602, + "loss": 2.5784, + "theoretical_loss": 3.9925332344326447, + "tokens_seen": 417792000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004411033099297894, + "loss": 3.0353, + "theoretical_loss": 3.9924636855567743, + "tokens_seen": 417857536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044109327983951857, + "loss": 2.7198, + "theoretical_loss": 3.992394150641636, + "tokens_seen": 417923072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044108324974924775, + "loss": 2.956, + "theoretical_loss": 3.992324629682238, + "tokens_seen": 417988608 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044107321965897693, + "loss": 2.4868, + "theoretical_loss": 3.992255122673593, + "tokens_seen": 418054144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410631895687061, + "loss": 3.0218, + "theoretical_loss": 3.9921856296107143, + "tokens_seen": 418119680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044105315947843535, + "loss": 2.8273, + "theoretical_loss": 3.992116150488619, + "tokens_seen": 418185216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410431293881645, + "loss": 2.8603, + "theoretical_loss": 3.992046685302327, + "tokens_seen": 418250752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004410330992978937, + "loss": 2.6586, + "theoretical_loss": 3.9919772340468596, + "tokens_seen": 418316288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044102306920762284, + "loss": 2.5768, + "theoretical_loss": 3.9919077967172414, + "tokens_seen": 418381824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044101303911735207, + "loss": 3.0208, + "theoretical_loss": 3.991838373308499, + "tokens_seen": 418447360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044100300902708125, + "loss": 3.1297, + "theoretical_loss": 3.9917689638156624, + "tokens_seen": 418512896 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044099297893681043, + "loss": 2.701, + "theoretical_loss": 3.9916995682337633, + "tokens_seen": 418578432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409829488465396, + "loss": 2.8845, + "theoretical_loss": 3.9916301865578365, + "tokens_seen": 418643968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044097291875626885, + "loss": 2.8086, + "theoretical_loss": 3.9915608187829195, + "tokens_seen": 418709504 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440962888665998, + "loss": 2.9099, + "theoretical_loss": 3.991491464904052, + "tokens_seen": 418775040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409528585757272, + "loss": 2.9136, + "theoretical_loss": 3.9914221249162756, + "tokens_seen": 418840576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044094282848545634, + "loss": 2.8021, + "theoretical_loss": 3.9913527988146353, + "tokens_seen": 418906112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409327983951856, + "loss": 2.8487, + "theoretical_loss": 3.9912834865941793, + "tokens_seen": 418971648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044092276830491476, + "loss": 2.9562, + "theoretical_loss": 3.9912141882499563, + "tokens_seen": 419037184 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044091273821464394, + "loss": 2.9475, + "theoretical_loss": 3.99114490377702, + "tokens_seen": 419102720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004409027081243731, + "loss": 2.7868, + "theoretical_loss": 3.991075633170425, + "tokens_seen": 419168256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408926780341023, + "loss": 2.9248, + "theoretical_loss": 3.991006376425228, + "tokens_seen": 419233792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408826479438315, + "loss": 2.8554, + "theoretical_loss": 3.99093713353649, + "tokens_seen": 419299328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408726178535607, + "loss": 2.6956, + "theoretical_loss": 3.9908679044992725, + "tokens_seen": 419364864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 242971, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7546820640563965, + "objective/train/theoretical_loss": 3.990798689308642, + "objective/train/tokens_used": 439890400, + "theoretical_loss": 3.990798689308642, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044086258776328984, + "loss": 2.9533, + "theoretical_loss": 3.990798689308642, + "tokens_seen": 419430400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408525576730191, + "loss": 3.0452, + "theoretical_loss": 3.9907294879596655, + "tokens_seen": 419495936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408425275827482, + "loss": 2.8135, + "theoretical_loss": 3.9906603004474137, + "tokens_seen": 419561472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044083249749247744, + "loss": 2.7665, + "theoretical_loss": 3.990591126766958, + "tokens_seen": 419627008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408224674022066, + "loss": 2.6103, + "theoretical_loss": 3.9905219669133745, + "tokens_seen": 419692544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004408124373119358, + "loss": 3.1082, + "theoretical_loss": 3.9904528208817407, + "tokens_seen": 419758080 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440802407221665, + "loss": 2.7595, + "theoretical_loss": 3.990383688667137, + "tokens_seen": 419823616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407923771313942, + "loss": 3.0086, + "theoretical_loss": 3.9903145702646468, + "tokens_seen": 419889152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044078234704112335, + "loss": 2.8238, + "theoretical_loss": 3.9902454656693545, + "tokens_seen": 419954688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407723169508526, + "loss": 2.9094, + "theoretical_loss": 3.990176374876348, + "tokens_seen": 420020224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407622868605817, + "loss": 3.11, + "theoretical_loss": 3.9901072978807184, + "tokens_seen": 420085760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044075225677031094, + "loss": 2.8017, + "theoretical_loss": 3.9900382346775576, + "tokens_seen": 420151296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407422266800401, + "loss": 3.0109, + "theoretical_loss": 3.9899691852619616, + "tokens_seen": 420216832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407321965897693, + "loss": 3.0303, + "theoretical_loss": 3.9899001496290274, + "tokens_seen": 420282368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004407221664994985, + "loss": 2.7646, + "theoretical_loss": 3.989831127773856, + "tokens_seen": 420347904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044071213640922767, + "loss": 2.8605, + "theoretical_loss": 3.989762119691551, + "tokens_seen": 420413440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044070210631895685, + "loss": 2.8231, + "theoretical_loss": 3.9896931253772165, + "tokens_seen": 420478976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406920762286861, + "loss": 2.5455, + "theoretical_loss": 3.989624144825961, + "tokens_seen": 420544512 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044068204613841527, + "loss": 3.1163, + "theoretical_loss": 3.9895551780328953, + "tokens_seen": 420610048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044067201604814445, + "loss": 2.6076, + "theoretical_loss": 3.989486224993131, + "tokens_seen": 420675584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406619859578737, + "loss": 3.1176, + "theoretical_loss": 3.9894172857017844, + "tokens_seen": 420741120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406519558676028, + "loss": 3.0003, + "theoretical_loss": 3.989348360153974, + "tokens_seen": 420806656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044064192577733204, + "loss": 2.955, + "theoretical_loss": 3.989279448344818, + "tokens_seen": 420872192 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044063189568706117, + "loss": 2.8199, + "theoretical_loss": 3.9892105502694415, + "tokens_seen": 420937728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406218655967904, + "loss": 2.9728, + "theoretical_loss": 3.989141665922969, + "tokens_seen": 421003264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 244183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9032411575317383, + "objective/train/theoretical_loss": 3.989072795300528, + "objective/train/tokens_used": 441528800, + "theoretical_loss": 3.989072795300528, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004406118355065196, + "loss": 2.8608, + "theoretical_loss": 3.989072795300528, + "tokens_seen": 421068800 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044060180541624877, + "loss": 2.9812, + "theoretical_loss": 3.989003938397249, + "tokens_seen": 421134336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044059177532597795, + "loss": 2.9128, + "theoretical_loss": 3.988935095208266, + "tokens_seen": 421199872 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044058174523570713, + "loss": 2.8998, + "theoretical_loss": 3.988866265728712, + "tokens_seen": 421265408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405717151454363, + "loss": 2.7796, + "theoretical_loss": 3.9887974499537266, + "tokens_seen": 421330944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044056168505516555, + "loss": 2.9465, + "theoretical_loss": 3.9887286478784496, + "tokens_seen": 421396480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405516549648947, + "loss": 2.6366, + "theoretical_loss": 3.988659859498023, + "tokens_seen": 421462016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004405416248746239, + "loss": 2.8122, + "theoretical_loss": 3.9885910848075934, + "tokens_seen": 421527552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044053159478435304, + "loss": 2.9463, + "theoretical_loss": 3.9885223238023073, + "tokens_seen": 421593088 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044052156469408227, + "loss": 3.0525, + "theoretical_loss": 3.988453576477315, + "tokens_seen": 421658624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044051153460381145, + "loss": 2.7746, + "theoretical_loss": 3.9883848428277697, + "tokens_seen": 421724160 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044050150451354063, + "loss": 2.7614, + "theoretical_loss": 3.988316122848826, + "tokens_seen": 421789696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404914744232698, + "loss": 2.9223, + "theoretical_loss": 3.988247416535642, + "tokens_seen": 421855232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044048144433299905, + "loss": 2.9403, + "theoretical_loss": 3.9881787238833772, + "tokens_seen": 421920768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404714142427282, + "loss": 2.7421, + "theoretical_loss": 3.9881100448871942, + "tokens_seen": 421986304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404613841524574, + "loss": 2.939, + "theoretical_loss": 3.988041379542258, + "tokens_seen": 422051840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044045135406218654, + "loss": 2.831, + "theoretical_loss": 3.9879727278437365, + "tokens_seen": 422117376 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404413239719158, + "loss": 2.8991, + "theoretical_loss": 3.987904089786799, + "tokens_seen": 422182912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044043129388164496, + "loss": 2.7653, + "theoretical_loss": 3.9878354653666177, + "tokens_seen": 422248448 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044042126379137414, + "loss": 2.9397, + "theoretical_loss": 3.987766854578368, + "tokens_seen": 422313984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404112337011033, + "loss": 3.1411, + "theoretical_loss": 3.987698257417227, + "tokens_seen": 422379520 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004404012036108325, + "loss": 3.0797, + "theoretical_loss": 3.9876296738783745, + "tokens_seen": 422445056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403911735205617, + "loss": 2.7101, + "theoretical_loss": 3.987561103956992, + "tokens_seen": 422510592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403811434302909, + "loss": 2.8034, + "theoretical_loss": 3.987492547648265, + "tokens_seen": 422576128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044037111334002004, + "loss": 2.8983, + "theoretical_loss": 3.98742400494738, + "tokens_seen": 422641664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 244758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9229331016540527, + "objective/train/theoretical_loss": 3.987355475849527, + "objective/train/tokens_used": 443167200, + "theoretical_loss": 3.987355475849527, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403610832497493, + "loss": 3.0326, + "theoretical_loss": 3.987355475849527, + "tokens_seen": 422707200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403510531594784, + "loss": 2.7311, + "theoretical_loss": 3.9872869603498975, + "tokens_seen": 422772736 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044034102306920764, + "loss": 2.9594, + "theoretical_loss": 3.987218458443686, + "tokens_seen": 422838272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403309929789368, + "loss": 2.9823, + "theoretical_loss": 3.9871499701260897, + "tokens_seen": 422903808 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440320962888666, + "loss": 2.9567, + "theoretical_loss": 3.9870814953923075, + "tokens_seen": 422969344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403109327983952, + "loss": 2.9457, + "theoretical_loss": 3.987013034237541, + "tokens_seen": 423034880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004403009027081244, + "loss": 2.9932, + "theoretical_loss": 3.9869445866569944, + "tokens_seen": 423100416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044029087261785355, + "loss": 2.9196, + "theoretical_loss": 3.986876152645875, + "tokens_seen": 423165952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402808425275828, + "loss": 2.7942, + "theoretical_loss": 3.9868077321993916, + "tokens_seen": 423231488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402708124373119, + "loss": 2.8662, + "theoretical_loss": 3.986739325312755, + "tokens_seen": 423297024 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044026078234704114, + "loss": 2.9557, + "theoretical_loss": 3.98667093198118, + "tokens_seen": 423362560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402507522567703, + "loss": 3.1629, + "theoretical_loss": 3.986602552199882, + "tokens_seen": 423428096 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402407221664995, + "loss": 2.9203, + "theoretical_loss": 3.9865341859640804, + "tokens_seen": 423493632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402306920762287, + "loss": 3.0048, + "theoretical_loss": 3.986465833268996, + "tokens_seen": 423559168 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044022066198595787, + "loss": 2.7389, + "theoretical_loss": 3.986397494109853, + "tokens_seen": 423624704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044021063189568705, + "loss": 2.9356, + "theoretical_loss": 3.986329168481877, + "tokens_seen": 423690240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004402006018054163, + "loss": 2.8892, + "theoretical_loss": 3.986260856380296, + "tokens_seen": 423755776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401905717151454, + "loss": 2.799, + "theoretical_loss": 3.9861925578003428, + "tokens_seen": 423821312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044018054162487465, + "loss": 2.9222, + "theoretical_loss": 3.986124272737248, + "tokens_seen": 423886848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044017051153460383, + "loss": 2.98, + "theoretical_loss": 3.986056001186249, + "tokens_seen": 423952384 + }, + { + "epoch": 0.13, + "learning_rate": 0.000440160481444333, + "loss": 2.9878, + "theoretical_loss": 3.9859877431425836, + "tokens_seen": 424017920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401504513540622, + "loss": 2.935, + "theoretical_loss": 3.9859194986014925, + "tokens_seen": 424083456 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044014042126379137, + "loss": 2.8919, + "theoretical_loss": 3.9858512675582185, + "tokens_seen": 424148992 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044013039117352055, + "loss": 2.856, + "theoretical_loss": 3.9857830500080067, + "tokens_seen": 424214528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401203610832498, + "loss": 2.9074, + "theoretical_loss": 3.985714845946105, + "tokens_seen": 424280064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 245792, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5828306674957275, + "objective/train/theoretical_loss": 3.9856466553677645, + "objective/train/tokens_used": 444805600, + "theoretical_loss": 3.9856466553677645, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004401103309929789, + "loss": 2.7627, + "theoretical_loss": 3.9856466553677645, + "tokens_seen": 424345600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044010030090270815, + "loss": 2.9771, + "theoretical_loss": 3.985578478268236, + "tokens_seen": 424411136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400902708124373, + "loss": 2.7067, + "theoretical_loss": 3.985510314642776, + "tokens_seen": 424476672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400802407221665, + "loss": 3.1371, + "theoretical_loss": 3.985442164486641, + "tokens_seen": 424542208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400702106318957, + "loss": 2.7251, + "theoretical_loss": 3.9853740277950918, + "tokens_seen": 424607744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400601805416249, + "loss": 2.7574, + "theoretical_loss": 3.98530590456339, + "tokens_seen": 424673280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044005015045135406, + "loss": 2.8466, + "theoretical_loss": 3.985237794786799, + "tokens_seen": 424738816 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044004012036108324, + "loss": 3.1828, + "theoretical_loss": 3.985169698460588, + "tokens_seen": 424804352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400300902708124, + "loss": 2.867, + "theoretical_loss": 3.9851016155800254, + "tokens_seen": 424869888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044002006018054165, + "loss": 2.7883, + "theoretical_loss": 3.9850335461403823, + "tokens_seen": 424935424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004400100300902708, + "loss": 2.8638, + "theoretical_loss": 3.984965490136934, + "tokens_seen": 425000960 + }, + { + "epoch": 0.13, + "learning_rate": 0.00044, + "loss": 2.989, + "theoretical_loss": 3.9848974475649563, + "tokens_seen": 425066496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399899699097292, + "loss": 3.2045, + "theoretical_loss": 3.9848294184197286, + "tokens_seen": 425132032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399799398194584, + "loss": 2.8111, + "theoretical_loss": 3.984761402696532, + "tokens_seen": 425197568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043996990972918756, + "loss": 2.8473, + "theoretical_loss": 3.9846934003906505, + "tokens_seen": 425263104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043995987963891674, + "loss": 2.684, + "theoretical_loss": 3.98462541149737, + "tokens_seen": 425328640 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399498495486459, + "loss": 2.6684, + "theoretical_loss": 3.984557436011978, + "tokens_seen": 425394176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043993981945837516, + "loss": 2.8611, + "theoretical_loss": 3.9844894739297674, + "tokens_seen": 425459712 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043992978936810434, + "loss": 2.8828, + "theoretical_loss": 3.98442152524603, + "tokens_seen": 425525248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399197592778335, + "loss": 2.7737, + "theoretical_loss": 3.984353589956062, + "tokens_seen": 425590784 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004399097291875627, + "loss": 3.0198, + "theoretical_loss": 3.9842856680551604, + "tokens_seen": 425656320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398996990972919, + "loss": 2.966, + "theoretical_loss": 3.9842177595386277, + "tokens_seen": 425721856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398896690070211, + "loss": 2.9014, + "theoretical_loss": 3.9841498644017643, + "tokens_seen": 425787392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043987963891675024, + "loss": 2.9488, + "theoretical_loss": 3.984081982639877, + "tokens_seen": 425852928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398696088264795, + "loss": 2.9806, + "theoretical_loss": 3.9840141142482723, + "tokens_seen": 425918464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 246485, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.934349298477173, + "objective/train/theoretical_loss": 3.98394625922226, + "objective/train/tokens_used": 446444000, + "theoretical_loss": 3.98394625922226, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398595787362086, + "loss": 2.8034, + "theoretical_loss": 3.98394625922226, + "tokens_seen": 425984000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043984954864593784, + "loss": 2.9262, + "theoretical_loss": 3.983878417557153, + "tokens_seen": 426049536 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439839518555667, + "loss": 2.9786, + "theoretical_loss": 3.9838105892482663, + "tokens_seen": 426115072 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398294884653962, + "loss": 2.6863, + "theoretical_loss": 3.9837427742909153, + "tokens_seen": 426180608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398194583751254, + "loss": 2.749, + "theoretical_loss": 3.9836749726804204, + "tokens_seen": 426246144 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004398094282848546, + "loss": 2.8953, + "theoretical_loss": 3.983607184412103, + "tokens_seen": 426311680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043979939819458375, + "loss": 2.841, + "theoretical_loss": 3.9835394094812875, + "tokens_seen": 426377216 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439789368104313, + "loss": 2.7696, + "theoretical_loss": 3.9834716478832997, + "tokens_seen": 426442752 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397793380140421, + "loss": 2.7188, + "theoretical_loss": 3.9834038996134686, + "tokens_seen": 426508288 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043976930792377134, + "loss": 2.717, + "theoretical_loss": 3.9833361646671257, + "tokens_seen": 426573824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397592778335005, + "loss": 2.977, + "theoretical_loss": 3.9832684430396035, + "tokens_seen": 426639360 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397492477432297, + "loss": 2.9569, + "theoretical_loss": 3.983200734726238, + "tokens_seen": 426704896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397392176529589, + "loss": 2.6681, + "theoretical_loss": 3.9831330397223685, + "tokens_seen": 426770432 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043972918756268807, + "loss": 2.8142, + "theoretical_loss": 3.9830653580233344, + "tokens_seen": 426835968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043971915747241725, + "loss": 2.796, + "theoretical_loss": 3.9829976896244794, + "tokens_seen": 426901504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004397091273821465, + "loss": 2.91, + "theoretical_loss": 3.9829300345211474, + "tokens_seen": 426967040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396990972918756, + "loss": 2.8034, + "theoretical_loss": 3.9828623927086872, + "tokens_seen": 427032576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043968906720160485, + "loss": 2.6024, + "theoretical_loss": 3.9827947641824473, + "tokens_seen": 427098112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043967903711133403, + "loss": 2.9885, + "theoretical_loss": 3.982727148937782, + "tokens_seen": 427163648 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396690070210632, + "loss": 3.0306, + "theoretical_loss": 3.982659546970044, + "tokens_seen": 427229184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396589769307924, + "loss": 2.9307, + "theoretical_loss": 3.9825919582745906, + "tokens_seen": 427294720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043964894684052157, + "loss": 3.1782, + "theoretical_loss": 3.9825243828467816, + "tokens_seen": 427360256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043963891675025075, + "loss": 2.7458, + "theoretical_loss": 3.982456820681978, + "tokens_seen": 427425792 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043962888665998, + "loss": 2.9876, + "theoretical_loss": 3.982389271775544, + "tokens_seen": 427491328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004396188565697091, + "loss": 2.8233, + "theoretical_loss": 3.9823217361228456, + "tokens_seen": 427556864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 247615, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7162137031555176, + "objective/train/theoretical_loss": 3.982254213719252, + "objective/train/tokens_used": 448082400, + "theoretical_loss": 3.982254213719252, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043960882647943835, + "loss": 2.8383, + "theoretical_loss": 3.982254213719252, + "tokens_seen": 427622400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395987963891675, + "loss": 2.8669, + "theoretical_loss": 3.9821867045601325, + "tokens_seen": 427687936 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395887662988967, + "loss": 2.893, + "theoretical_loss": 3.9821192086408623, + "tokens_seen": 427753472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395787362086259, + "loss": 2.8802, + "theoretical_loss": 3.982051725956816, + "tokens_seen": 427819008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395687061183551, + "loss": 2.587, + "theoretical_loss": 3.9819842565033703, + "tokens_seen": 427884544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043955867602808426, + "loss": 2.9512, + "theoretical_loss": 3.9819168002759078, + "tokens_seen": 427950080 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043954864593781344, + "loss": 2.7715, + "theoretical_loss": 3.9818493572698093, + "tokens_seen": 428015616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395386158475426, + "loss": 3.0963, + "theoretical_loss": 3.9817819274804602, + "tokens_seen": 428081152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043952858575727185, + "loss": 2.8617, + "theoretical_loss": 3.9817145109032475, + "tokens_seen": 428146688 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439518555667001, + "loss": 2.8287, + "theoretical_loss": 3.9816471075335604, + "tokens_seen": 428212224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004395085255767302, + "loss": 2.9118, + "theoretical_loss": 3.981579717366791, + "tokens_seen": 428277760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394984954864594, + "loss": 3.141, + "theoretical_loss": 3.981512340398333, + "tokens_seen": 428343296 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394884653961886, + "loss": 2.6428, + "theoretical_loss": 3.9814449766235835, + "tokens_seen": 428408832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043947843530591776, + "loss": 2.9725, + "theoretical_loss": 3.9813776260379408, + "tokens_seen": 428474368 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043946840521564694, + "loss": 2.7616, + "theoretical_loss": 3.9813102886368057, + "tokens_seen": 428539904 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394583751253761, + "loss": 2.8903, + "theoretical_loss": 3.9812429644155816, + "tokens_seen": 428605440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043944834503510536, + "loss": 3.0138, + "theoretical_loss": 3.9811756533696743, + "tokens_seen": 428670976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394383149448345, + "loss": 2.93, + "theoretical_loss": 3.9811083554944915, + "tokens_seen": 428736512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394282848545637, + "loss": 2.7467, + "theoretical_loss": 3.9810410707854436, + "tokens_seen": 428802048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043941825476429285, + "loss": 3.1358, + "theoretical_loss": 3.980973799237943, + "tokens_seen": 428867584 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004394082246740221, + "loss": 2.8636, + "theoretical_loss": 3.9809065408474047, + "tokens_seen": 428933120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043939819458375126, + "loss": 3.0446, + "theoretical_loss": 3.980839295609246, + "tokens_seen": 428998656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043938816449348044, + "loss": 2.7096, + "theoretical_loss": 3.9807720635188857, + "tokens_seen": 429064192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393781344032096, + "loss": 2.8974, + "theoretical_loss": 3.9807048445717457, + "tokens_seen": 429129728 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393681043129388, + "loss": 3.0659, + "theoretical_loss": 3.9806376387632505, + "tokens_seen": 429195264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 248896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7603349685668945, + "objective/train/theoretical_loss": 3.9805704460888256, + "objective/train/tokens_used": 449720800, + "theoretical_loss": 3.9805704460888256, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439358074222668, + "loss": 2.9696, + "theoretical_loss": 3.9805704460888256, + "tokens_seen": 429260800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393480441323972, + "loss": 3.04, + "theoretical_loss": 3.9805032665439004, + "tokens_seen": 429326336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043933801404212635, + "loss": 2.7513, + "theoretical_loss": 3.9804361001239057, + "tokens_seen": 429391872 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004393279839518556, + "loss": 2.8466, + "theoretical_loss": 3.9803689468242744, + "tokens_seen": 429457408 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043931795386158477, + "loss": 2.7847, + "theoretical_loss": 3.9803018066404414, + "tokens_seen": 429522944 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043930792377131395, + "loss": 2.9597, + "theoretical_loss": 3.980234679567845, + "tokens_seen": 429588480 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043929789368104313, + "loss": 2.878, + "theoretical_loss": 3.980167565601925, + "tokens_seen": 429654016 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392878635907723, + "loss": 3.0399, + "theoretical_loss": 3.9801004647381246, + "tokens_seen": 429719552 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392778335005015, + "loss": 3.145, + "theoretical_loss": 3.980033376971887, + "tokens_seen": 429785088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392678034102307, + "loss": 3.0189, + "theoretical_loss": 3.97996630229866, + "tokens_seen": 429850624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043925777331995985, + "loss": 2.8291, + "theoretical_loss": 3.9798992407138924, + "tokens_seen": 429916160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392477432296891, + "loss": 2.8082, + "theoretical_loss": 3.9798321922130357, + "tokens_seen": 429981696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392377131394182, + "loss": 2.8025, + "theoretical_loss": 3.979765156791543, + "tokens_seen": 430047232 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043922768304914745, + "loss": 2.8277, + "theoretical_loss": 3.979698134444871, + "tokens_seen": 430112768 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043921765295887663, + "loss": 2.9302, + "theoretical_loss": 3.979631125168478, + "tokens_seen": 430178304 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004392076228686058, + "loss": 2.7819, + "theoretical_loss": 3.979564128957824, + "tokens_seen": 430243840 + }, + { + "epoch": 0.13, + "learning_rate": 0.000439197592778335, + "loss": 2.887, + "theoretical_loss": 3.979497145808372, + "tokens_seen": 430309376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043918756268806423, + "loss": 2.8212, + "theoretical_loss": 3.979430175715587, + "tokens_seen": 430374912 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391775325977934, + "loss": 3.0824, + "theoretical_loss": 3.979363218674936, + "tokens_seen": 430440448 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391675025075226, + "loss": 2.8863, + "theoretical_loss": 3.9792962746818894, + "tokens_seen": 430505984 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043915747241725177, + "loss": 2.9136, + "theoretical_loss": 3.9792293437319177, + "tokens_seen": 430571520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043914744232698095, + "loss": 2.8946, + "theoretical_loss": 3.979162425820496, + "tokens_seen": 430637056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391374122367102, + "loss": 2.8152, + "theoretical_loss": 3.9790955209431007, + "tokens_seen": 430702592 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391273821464393, + "loss": 2.5009, + "theoretical_loss": 3.9790286290952097, + "tokens_seen": 430768128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043911735205616855, + "loss": 2.7643, + "theoretical_loss": 3.978961750272304, + "tokens_seen": 430833664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 249330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5802011489868164, + "objective/train/theoretical_loss": 3.978894884469867, + "objective/train/tokens_used": 451359200, + "theoretical_loss": 3.978894884469867, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004391073219658977, + "loss": 2.6646, + "theoretical_loss": 3.978894884469867, + "tokens_seen": 430899200 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390972918756269, + "loss": 2.8741, + "theoretical_loss": 3.978828031683384, + "tokens_seen": 430964736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390872617853561, + "loss": 3.135, + "theoretical_loss": 3.978761191908343, + "tokens_seen": 431030272 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390772316950853, + "loss": 2.7624, + "theoretical_loss": 3.9786943651402327, + "tokens_seen": 431095808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043906720160481446, + "loss": 2.8756, + "theoretical_loss": 3.9786275513745464, + "tokens_seen": 431161344 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043905717151454364, + "loss": 2.8513, + "theoretical_loss": 3.9785607506067775, + "tokens_seen": 431226880 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390471414242728, + "loss": 2.951, + "theoretical_loss": 3.978493962832424, + "tokens_seen": 431292416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043903711133400205, + "loss": 2.7171, + "theoretical_loss": 3.9784271880469824, + "tokens_seen": 431357952 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390270812437312, + "loss": 2.7085, + "theoretical_loss": 3.978360426245956, + "tokens_seen": 431423488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390170511534604, + "loss": 2.8883, + "theoretical_loss": 3.9782936774248476, + "tokens_seen": 431489024 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004390070210631896, + "loss": 2.7489, + "theoretical_loss": 3.978226941579162, + "tokens_seen": 431554560 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389969909729188, + "loss": 2.8769, + "theoretical_loss": 3.9781602187044074, + "tokens_seen": 431620096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043898696088264796, + "loss": 2.9323, + "theoretical_loss": 3.9780935087960945, + "tokens_seen": 431685632 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043897693079237714, + "loss": 2.8336, + "theoretical_loss": 3.978026811849735, + "tokens_seen": 431751168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389669007021063, + "loss": 2.4792, + "theoretical_loss": 3.977960127860843, + "tokens_seen": 431816704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043895687061183556, + "loss": 2.7761, + "theoretical_loss": 3.977893456824936, + "tokens_seen": 431882240 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389468405215647, + "loss": 2.847, + "theoretical_loss": 3.9778267987375324, + "tokens_seen": 431947776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389368104312939, + "loss": 2.9699, + "theoretical_loss": 3.9777601535941542, + "tokens_seen": 432013312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043892678034102305, + "loss": 2.9322, + "theoretical_loss": 3.977693521390324, + "tokens_seen": 432078848 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004389167502507523, + "loss": 2.7251, + "theoretical_loss": 3.9776269021215676, + "tokens_seen": 432144384 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043890672016048146, + "loss": 2.7903, + "theoretical_loss": 3.9775602957834133, + "tokens_seen": 432209920 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043889669007021064, + "loss": 3.0632, + "theoretical_loss": 3.977493702371391, + "tokens_seen": 432275456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388866599799398, + "loss": 2.9212, + "theoretical_loss": 3.977427121881033, + "tokens_seen": 432340992 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438876629889669, + "loss": 2.8574, + "theoretical_loss": 3.977360554307874, + "tokens_seen": 432406528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388665997993982, + "loss": 2.6996, + "theoretical_loss": 3.9772939996474506, + "tokens_seen": 432472064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 250327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1851444244384766, + "objective/train/theoretical_loss": 3.9772274578953013, + "objective/train/tokens_used": 452997600, + "theoretical_loss": 3.9772274578953013, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388565697091274, + "loss": 2.49, + "theoretical_loss": 3.9772274578953013, + "tokens_seen": 432537600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043884653961885655, + "loss": 3.0437, + "theoretical_loss": 3.977160929046968, + "tokens_seen": 432603136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004388365095285858, + "loss": 3.0128, + "theoretical_loss": 3.977094413097994, + "tokens_seen": 432668672 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043882647943831497, + "loss": 3.064, + "theoretical_loss": 3.9770279100439256, + "tokens_seen": 432734208 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043881644934804415, + "loss": 3.0597, + "theoretical_loss": 3.9769614198803094, + "tokens_seen": 432799744 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043880641925777333, + "loss": 2.7542, + "theoretical_loss": 3.976894942602696, + "tokens_seen": 432865280 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387963891675025, + "loss": 2.9004, + "theoretical_loss": 3.976828478206638, + "tokens_seen": 432930816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387863590772317, + "loss": 2.7999, + "theoretical_loss": 3.976762026687689, + "tokens_seen": 432996352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387763289869609, + "loss": 2.9483, + "theoretical_loss": 3.9766955880414065, + "tokens_seen": 433061888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043876629889669005, + "loss": 2.7133, + "theoretical_loss": 3.9766291622633494, + "tokens_seen": 433127424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387562688064193, + "loss": 2.8788, + "theoretical_loss": 3.9765627493490783, + "tokens_seen": 433192960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387462387161484, + "loss": 2.7892, + "theoretical_loss": 3.976496349294157, + "tokens_seen": 433258496 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043873620862587765, + "loss": 2.8578, + "theoretical_loss": 3.9764299620941506, + "tokens_seen": 433324032 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043872617853560683, + "loss": 2.86, + "theoretical_loss": 3.9763635877446273, + "tokens_seen": 433389568 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438716148445336, + "loss": 2.8879, + "theoretical_loss": 3.9762972262411562, + "tokens_seen": 433455104 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004387061183550652, + "loss": 2.9078, + "theoretical_loss": 3.97623087757931, + "tokens_seen": 433520640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043869608826479443, + "loss": 2.9007, + "theoretical_loss": 3.9761645417546627, + "tokens_seen": 433586176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043868605817452356, + "loss": 2.7336, + "theoretical_loss": 3.9760982187627913, + "tokens_seen": 433651712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386760280842528, + "loss": 3.0301, + "theoretical_loss": 3.976031908599274, + "tokens_seen": 433717248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386659979939819, + "loss": 2.8723, + "theoretical_loss": 3.9759656112596913, + "tokens_seen": 433782784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043865596790371115, + "loss": 2.9686, + "theoretical_loss": 3.9758993267396265, + "tokens_seen": 433848320 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043864593781344033, + "loss": 2.7986, + "theoretical_loss": 3.9758330550346663, + "tokens_seen": 433913856 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386359077231695, + "loss": 2.8748, + "theoretical_loss": 3.975766796140396, + "tokens_seen": 433979392 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386258776328987, + "loss": 2.8568, + "theoretical_loss": 3.975700550052406, + "tokens_seen": 434044928 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004386158475426279, + "loss": 2.828, + "theoretical_loss": 3.9756343167662886, + "tokens_seen": 434110464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 251069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.99489426612854, + "objective/train/theoretical_loss": 3.9755680962776374, + "objective/train/tokens_used": 454636000, + "theoretical_loss": 3.9755680962776374, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043860581745235706, + "loss": 2.7617, + "theoretical_loss": 3.9755680962776374, + "tokens_seen": 434176000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385957873620863, + "loss": 3.1528, + "theoretical_loss": 3.9755018885820483, + "tokens_seen": 434241536 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385857572718154, + "loss": 2.9258, + "theoretical_loss": 3.97543569367512, + "tokens_seen": 434307072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043857572718154466, + "loss": 2.9792, + "theoretical_loss": 3.9753695115524534, + "tokens_seen": 434372608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385656970912738, + "loss": 2.7644, + "theoretical_loss": 3.97530334220965, + "tokens_seen": 434438144 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438555667001003, + "loss": 2.9931, + "theoretical_loss": 3.975237185642316, + "tokens_seen": 434503680 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385456369107322, + "loss": 2.948, + "theoretical_loss": 3.9751710418460577, + "tokens_seen": 434569216 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385356068204614, + "loss": 3.1, + "theoretical_loss": 3.9751049108164844, + "tokens_seen": 434634752 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043852557673019056, + "loss": 2.8339, + "theoretical_loss": 3.9750387925492072, + "tokens_seen": 434700288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385155466399198, + "loss": 2.9765, + "theoretical_loss": 3.974972687039841, + "tokens_seen": 434765824 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004385055165496489, + "loss": 2.5896, + "theoretical_loss": 3.9749065942839996, + "tokens_seen": 434831360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043849548645937816, + "loss": 2.8618, + "theoretical_loss": 3.974840514277303, + "tokens_seen": 434896896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384854563691073, + "loss": 2.9577, + "theoretical_loss": 3.974774447015369, + "tokens_seen": 434962432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384754262788365, + "loss": 2.9189, + "theoretical_loss": 3.9747083924938216, + "tokens_seen": 435027968 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384653961885657, + "loss": 2.779, + "theoretical_loss": 3.974642350708284, + "tokens_seen": 435093504 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384553660982949, + "loss": 2.99, + "theoretical_loss": 3.974576321654384, + "tokens_seen": 435159040 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043844533600802407, + "loss": 2.6761, + "theoretical_loss": 3.9745103053277493, + "tokens_seen": 435224576 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043843530591775325, + "loss": 2.8512, + "theoretical_loss": 3.974444301724011, + "tokens_seen": 435290112 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004384252758274825, + "loss": 2.8829, + "theoretical_loss": 3.9743783108388024, + "tokens_seen": 435355648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043841524573721166, + "loss": 3.0582, + "theoretical_loss": 3.9743123326677585, + "tokens_seen": 435421184 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043840521564694084, + "loss": 3.0055, + "theoretical_loss": 3.9742463672065167, + "tokens_seen": 435486720 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043839518555667, + "loss": 2.7174, + "theoretical_loss": 3.9741804144507165, + "tokens_seen": 435552256 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383851554663992, + "loss": 2.9559, + "theoretical_loss": 3.974114474395999, + "tokens_seen": 435617792 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383751253761284, + "loss": 2.9737, + "theoretical_loss": 3.9740485470380094, + "tokens_seen": 435683328 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383650952858576, + "loss": 2.8755, + "theoretical_loss": 3.973982632372392, + "tokens_seen": 435748864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 252128, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7361905574798584, + "objective/train/theoretical_loss": 3.973916730394796, + "objective/train/tokens_used": 456274400, + "theoretical_loss": 3.973916730394796, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043835506519558675, + "loss": 2.7758, + "theoretical_loss": 3.973916730394796, + "tokens_seen": 435814400 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438345035105316, + "loss": 2.9473, + "theoretical_loss": 3.9738508411008713, + "tokens_seen": 435879936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043833500501504517, + "loss": 2.8355, + "theoretical_loss": 3.97378496448627, + "tokens_seen": 435945472 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043832497492477435, + "loss": 2.9717, + "theoretical_loss": 3.9737191005466475, + "tokens_seen": 436011008 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043831494483450353, + "loss": 2.8114, + "theoretical_loss": 3.973653249277659, + "tokens_seen": 436076544 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004383049147442327, + "loss": 2.9563, + "theoretical_loss": 3.973587410674965, + "tokens_seen": 436142080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382948846539619, + "loss": 3.0945, + "theoretical_loss": 3.973521584734225, + "tokens_seen": 436207616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382848545636911, + "loss": 3.0023, + "theoretical_loss": 3.9734557714511034, + "tokens_seen": 436273152 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043827482447342025, + "loss": 3.158, + "theoretical_loss": 3.9733899708212643, + "tokens_seen": 436338688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382647943831495, + "loss": 2.7855, + "theoretical_loss": 3.973324182840375, + "tokens_seen": 436404224 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382547642928786, + "loss": 2.7221, + "theoretical_loss": 3.9732584075041064, + "tokens_seen": 436469760 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043824473420260785, + "loss": 2.9749, + "theoretical_loss": 3.9731926448081287, + "tokens_seen": 436535296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043823470411233703, + "loss": 2.8126, + "theoretical_loss": 3.973126894748116, + "tokens_seen": 436600832 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382246740220662, + "loss": 2.909, + "theoretical_loss": 3.973061157319745, + "tokens_seen": 436666368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004382146439317954, + "loss": 2.9275, + "theoretical_loss": 3.972995432518693, + "tokens_seen": 436731904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043820461384152463, + "loss": 2.8421, + "theoretical_loss": 3.9729297203406393, + "tokens_seen": 436797440 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043819458375125376, + "loss": 2.7437, + "theoretical_loss": 3.9728640207812678, + "tokens_seen": 436862976 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438184553660983, + "loss": 2.7797, + "theoretical_loss": 3.9727983338362627, + "tokens_seen": 436928512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381745235707121, + "loss": 2.9771, + "theoretical_loss": 3.9727326595013093, + "tokens_seen": 436994048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043816449348044135, + "loss": 2.5017, + "theoretical_loss": 3.972666997772097, + "tokens_seen": 437059584 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043815446339017053, + "loss": 3.0961, + "theoretical_loss": 3.972601348644317, + "tokens_seen": 437125120 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381444332998997, + "loss": 2.7381, + "theoretical_loss": 3.9725357121136615, + "tokens_seen": 437190656 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381344032096289, + "loss": 2.9324, + "theoretical_loss": 3.9724700881758253, + "tokens_seen": 437256192 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381243731193581, + "loss": 2.7951, + "theoretical_loss": 3.9724044768265063, + "tokens_seen": 437321728 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043811434302908726, + "loss": 2.9698, + "theoretical_loss": 3.9723388780614033, + "tokens_seen": 437387264 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 252698, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6559126377105713, + "objective/train/theoretical_loss": 3.9722732918762174, + "objective/train/tokens_used": 457912800, + "theoretical_loss": 3.9722732918762174, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004381043129388165, + "loss": 2.88, + "theoretical_loss": 3.9722732918762174, + "tokens_seen": 437452800 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380942828485456, + "loss": 2.7562, + "theoretical_loss": 3.9722077182666524, + "tokens_seen": 437518336 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043808425275827486, + "loss": 2.9245, + "theoretical_loss": 3.9721421572284137, + "tokens_seen": 437583872 + }, + { + "epoch": 0.13, + "learning_rate": 0.000438074222668004, + "loss": 2.7057, + "theoretical_loss": 3.972076608757209, + "tokens_seen": 437649408 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380641925777332, + "loss": 3.2024, + "theoretical_loss": 3.9720110728487485, + "tokens_seen": 437714944 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380541624874624, + "loss": 2.9399, + "theoretical_loss": 3.9719455494987432, + "tokens_seen": 437780480 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380441323971916, + "loss": 2.8494, + "theoretical_loss": 3.971880038702908, + "tokens_seen": 437846016 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043803410230692076, + "loss": 3.0082, + "theoretical_loss": 3.9718145404569585, + "tokens_seen": 437911552 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043802407221665, + "loss": 2.8472, + "theoretical_loss": 3.971749054756613, + "tokens_seen": 437977088 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004380140421263791, + "loss": 2.8728, + "theoretical_loss": 3.971683581597592, + "tokens_seen": 438042624 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043800401203610836, + "loss": 2.8619, + "theoretical_loss": 3.9716181209756174, + "tokens_seen": 438108160 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379939819458375, + "loss": 2.8314, + "theoretical_loss": 3.9715526728864137, + "tokens_seen": 438173696 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379839518555667, + "loss": 2.4957, + "theoretical_loss": 3.971487237325708, + "tokens_seen": 438239232 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379739217652959, + "loss": 2.8016, + "theoretical_loss": 3.9714218142892292, + "tokens_seen": 438304768 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379638916750251, + "loss": 2.8969, + "theoretical_loss": 3.9713564037727074, + "tokens_seen": 438370304 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043795386158475427, + "loss": 2.6609, + "theoretical_loss": 3.9712910057718758, + "tokens_seen": 438435840 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043794383149448345, + "loss": 3.0509, + "theoretical_loss": 3.9712256202824694, + "tokens_seen": 438501376 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043793380140421263, + "loss": 2.8158, + "theoretical_loss": 3.9711602473002245, + "tokens_seen": 438566912 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043792377131394186, + "loss": 2.9429, + "theoretical_loss": 3.9710948868208815, + "tokens_seen": 438632448 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437913741223671, + "loss": 2.6593, + "theoretical_loss": 3.971029538840181, + "tokens_seen": 438697984 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004379037111334002, + "loss": 2.7673, + "theoretical_loss": 3.9709642033538666, + "tokens_seen": 438763520 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043789368104312935, + "loss": 2.89, + "theoretical_loss": 3.970898880357683, + "tokens_seen": 438829056 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378836509528586, + "loss": 2.8787, + "theoretical_loss": 3.9708335698473785, + "tokens_seen": 438894592 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043787362086258777, + "loss": 2.8641, + "theoretical_loss": 3.9707682718187023, + "tokens_seen": 438960128 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043786359077231695, + "loss": 2.7491, + "theoretical_loss": 3.9707029862674066, + "tokens_seen": 439025664 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 253958, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.985783815383911, + "objective/train/theoretical_loss": 3.970637713189244, + "objective/train/tokens_used": 459551200, + "theoretical_loss": 3.970637713189244, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043785356068204613, + "loss": 2.9894, + "theoretical_loss": 3.970637713189244, + "tokens_seen": 439091200 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043784353059177537, + "loss": 2.6916, + "theoretical_loss": 3.9705724525799715, + "tokens_seen": 439156736 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378335005015045, + "loss": 2.6918, + "theoretical_loss": 3.9705072044353455, + "tokens_seen": 439222272 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043782347041123373, + "loss": 2.917, + "theoretical_loss": 3.970441968751128, + "tokens_seen": 439287808 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043781344032096286, + "loss": 2.8387, + "theoretical_loss": 3.9703767455230796, + "tokens_seen": 439353344 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004378034102306921, + "loss": 2.9011, + "theoretical_loss": 3.970311534746965, + "tokens_seen": 439418880 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043779338014042127, + "loss": 2.774, + "theoretical_loss": 3.97024633641855, + "tokens_seen": 439484416 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043778335005015045, + "loss": 2.9159, + "theoretical_loss": 3.970181150533603, + "tokens_seen": 439549952 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043777331995987963, + "loss": 2.7145, + "theoretical_loss": 3.970115977087895, + "tokens_seen": 439615488 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377632898696088, + "loss": 2.9115, + "theoretical_loss": 3.9700508160771966, + "tokens_seen": 439681024 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437753259779338, + "loss": 2.9576, + "theoretical_loss": 3.9699856674972844, + "tokens_seen": 439746560 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043774322968906723, + "loss": 3.0514, + "theoretical_loss": 3.969920531343934, + "tokens_seen": 439812096 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043773319959879636, + "loss": 2.7901, + "theoretical_loss": 3.9698554076129233, + "tokens_seen": 439877632 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377231695085256, + "loss": 2.6878, + "theoretical_loss": 3.969790296300034, + "tokens_seen": 439943168 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004377131394182547, + "loss": 2.7006, + "theoretical_loss": 3.969725197401049, + "tokens_seen": 440008704 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043770310932798396, + "loss": 2.9952, + "theoretical_loss": 3.969660110911752, + "tokens_seen": 440074240 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043769307923771314, + "loss": 3.0273, + "theoretical_loss": 3.96959503682793, + "tokens_seen": 440139776 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376830491474423, + "loss": 2.8135, + "theoretical_loss": 3.969529975145372, + "tokens_seen": 440205312 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043767301905717155, + "loss": 2.7669, + "theoretical_loss": 3.9694649258598704, + "tokens_seen": 440270848 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043766298896690073, + "loss": 2.767, + "theoretical_loss": 3.9693998889672164, + "tokens_seen": 440336384 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376529588766299, + "loss": 2.7824, + "theoretical_loss": 3.969334864463206, + "tokens_seen": 440401920 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376429287863591, + "loss": 2.6347, + "theoretical_loss": 3.9692698523436354, + "tokens_seen": 440467456 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376328986960883, + "loss": 2.8143, + "theoretical_loss": 3.9692048526043044, + "tokens_seen": 440532992 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043762286860581746, + "loss": 3.0135, + "theoretical_loss": 3.969139865241014, + "tokens_seen": 440598528 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376128385155467, + "loss": 2.7171, + "theoretical_loss": 3.9690748902495683, + "tokens_seen": 440664064 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 254624, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.297787666320801, + "objective/train/theoretical_loss": 3.9690099276257715, + "objective/train/tokens_used": 461189600, + "theoretical_loss": 3.9690099276257715, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004376028084252758, + "loss": 2.9859, + "theoretical_loss": 3.9690099276257715, + "tokens_seen": 440729600 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043759277833500506, + "loss": 2.8268, + "theoretical_loss": 3.9689449773654317, + "tokens_seen": 440795136 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375827482447342, + "loss": 2.5088, + "theoretical_loss": 3.9688800394643575, + "tokens_seen": 440860672 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375727181544634, + "loss": 2.7787, + "theoretical_loss": 3.968815113918361, + "tokens_seen": 440926208 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375626880641926, + "loss": 2.9048, + "theoretical_loss": 3.968750200723256, + "tokens_seen": 440991744 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375526579739218, + "loss": 2.8828, + "theoretical_loss": 3.9686852998748567, + "tokens_seen": 441057280 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043754262788365096, + "loss": 2.6297, + "theoretical_loss": 3.968620411368982, + "tokens_seen": 441122816 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375325977933802, + "loss": 3.0614, + "theoretical_loss": 3.9685555352014505, + "tokens_seen": 441188352 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375225677031093, + "loss": 2.958, + "theoretical_loss": 3.9684906713680848, + "tokens_seen": 441253888 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043751253761283856, + "loss": 2.84, + "theoretical_loss": 3.9684258198647075, + "tokens_seen": 441319424 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004375025075225677, + "loss": 2.816, + "theoretical_loss": 3.9683609806871454, + "tokens_seen": 441384960 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374924774322969, + "loss": 2.809, + "theoretical_loss": 3.9682961538312256, + "tokens_seen": 441450496 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374824473420261, + "loss": 2.9887, + "theoretical_loss": 3.968231339292778, + "tokens_seen": 441516032 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374724172517553, + "loss": 2.8036, + "theoretical_loss": 3.968166537067634, + "tokens_seen": 441581568 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043746238716148447, + "loss": 2.7846, + "theoretical_loss": 3.9681017471516284, + "tokens_seen": 441647104 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043745235707121365, + "loss": 2.7036, + "theoretical_loss": 3.968036969540596, + "tokens_seen": 441712640 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043744232698094283, + "loss": 2.7011, + "theoretical_loss": 3.9679722042303753, + "tokens_seen": 441778176 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043743229689067206, + "loss": 2.802, + "theoretical_loss": 3.967907451216806, + "tokens_seen": 441843712 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374222668004012, + "loss": 2.9564, + "theoretical_loss": 3.9678427104957303, + "tokens_seen": 441909248 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004374122367101304, + "loss": 2.9014, + "theoretical_loss": 3.9677779820629917, + "tokens_seen": 441974784 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043740220661985955, + "loss": 2.8714, + "theoretical_loss": 3.9677132659144365, + "tokens_seen": 442040320 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373921765295888, + "loss": 2.8678, + "theoretical_loss": 3.9676485620459125, + "tokens_seen": 442105856 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043738214643931797, + "loss": 2.789, + "theoretical_loss": 3.96758387045327, + "tokens_seen": 442171392 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043737211634904715, + "loss": 2.6459, + "theoretical_loss": 3.967519191132361, + "tokens_seen": 442236928 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043736208625877633, + "loss": 3.0265, + "theoretical_loss": 3.9674545240790393, + "tokens_seen": 442302464 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 255794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5010974407196045, + "objective/train/theoretical_loss": 3.967389869289161, + "objective/train/tokens_used": 462828000, + "theoretical_loss": 3.967389869289161, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043735205616850557, + "loss": 2.7137, + "theoretical_loss": 3.967389869289161, + "tokens_seen": 442368000 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373420260782347, + "loss": 2.6839, + "theoretical_loss": 3.967325226758584, + "tokens_seen": 442433536 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043733199598796393, + "loss": 2.985, + "theoretical_loss": 3.967260596483169, + "tokens_seen": 442499072 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043732196589769306, + "loss": 2.8362, + "theoretical_loss": 3.9671959784587774, + "tokens_seen": 442564608 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004373119358074223, + "loss": 2.7015, + "theoretical_loss": 3.9671313726812736, + "tokens_seen": 442630144 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043730190571715147, + "loss": 2.957, + "theoretical_loss": 3.967066779146524, + "tokens_seen": 442695680 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043729187562688065, + "loss": 2.9622, + "theoretical_loss": 3.9670021978503964, + "tokens_seen": 442761216 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043728184553660983, + "loss": 2.7419, + "theoretical_loss": 3.966937628788761, + "tokens_seen": 442826752 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437271815446339, + "loss": 2.9077, + "theoretical_loss": 3.96687307195749, + "tokens_seen": 442892288 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372617853560682, + "loss": 2.4599, + "theoretical_loss": 3.966808527352457, + "tokens_seen": 442957824 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043725175526579743, + "loss": 2.9802, + "theoretical_loss": 3.966743994969539, + "tokens_seen": 443023360 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043724172517552656, + "loss": 2.8803, + "theoretical_loss": 3.9666794748046135, + "tokens_seen": 443088896 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372316950852558, + "loss": 2.752, + "theoretical_loss": 3.9666149668535615, + "tokens_seen": 443154432 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004372216649949849, + "loss": 2.9157, + "theoretical_loss": 3.9665504711122637, + "tokens_seen": 443219968 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043721163490471416, + "loss": 2.8042, + "theoretical_loss": 3.9664859875766054, + "tokens_seen": 443285504 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043720160481444334, + "loss": 2.7464, + "theoretical_loss": 3.9664215162424723, + "tokens_seen": 443351040 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371915747241725, + "loss": 2.9279, + "theoretical_loss": 3.9663570571057525, + "tokens_seen": 443416576 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371815446339017, + "loss": 2.8611, + "theoretical_loss": 3.9662926101623364, + "tokens_seen": 443482112 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043717151454363093, + "loss": 3.013, + "theoretical_loss": 3.9662281754081157, + "tokens_seen": 443547648 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043716148445336006, + "loss": 2.9245, + "theoretical_loss": 3.9661637528389853, + "tokens_seen": 443613184 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371514543630893, + "loss": 2.8922, + "theoretical_loss": 3.96609934245084, + "tokens_seen": 443678720 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371414242728184, + "loss": 2.6798, + "theoretical_loss": 3.966034944239579, + "tokens_seen": 443744256 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043713139418254766, + "loss": 2.6617, + "theoretical_loss": 3.965970558201102, + "tokens_seen": 443809792 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043712136409227684, + "loss": 2.8185, + "theoretical_loss": 3.9659061843313106, + "tokens_seen": 443875328 + }, + { + "epoch": 0.13, + "learning_rate": 0.000437111334002006, + "loss": 2.7108, + "theoretical_loss": 3.9658418226261096, + "tokens_seen": 443940864 + }, + { + "epoch": 0.13, + "objective/train/docs_used": 257001, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6992599964141846, + "objective/train/theoretical_loss": 3.9657774730814044, + "objective/train/tokens_used": 464466400, + "theoretical_loss": 3.9657774730814044, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004371013039117352, + "loss": 2.7786, + "theoretical_loss": 3.9657774730814044, + "tokens_seen": 444006400 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370912738214644, + "loss": 3.0641, + "theoretical_loss": 3.965713135693103, + "tokens_seen": 444071936 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043708124373119356, + "loss": 2.8226, + "theoretical_loss": 3.9656488104571155, + "tokens_seen": 444137472 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370712136409228, + "loss": 2.8475, + "theoretical_loss": 3.965584497369355, + "tokens_seen": 444203008 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370611835506519, + "loss": 2.7757, + "theoretical_loss": 3.9655201964257336, + "tokens_seen": 444268544 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043705115346038116, + "loss": 2.6841, + "theoretical_loss": 3.965455907622168, + "tokens_seen": 444334080 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370411233701103, + "loss": 2.8601, + "theoretical_loss": 3.9653916309545765, + "tokens_seen": 444399616 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370310932798395, + "loss": 2.8329, + "theoretical_loss": 3.9653273664188786, + "tokens_seen": 444465152 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370210631895687, + "loss": 2.9373, + "theoretical_loss": 3.9652631140109955, + "tokens_seen": 444530688 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004370110330992979, + "loss": 2.8243, + "theoretical_loss": 3.965198873726852, + "tokens_seen": 444596224 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043700100300902707, + "loss": 3.0494, + "theoretical_loss": 3.965134645562374, + "tokens_seen": 444661760 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369909729187563, + "loss": 3.087, + "theoretical_loss": 3.9650704295134878, + "tokens_seen": 444727296 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043698094282848543, + "loss": 2.9131, + "theoretical_loss": 3.9650062255761247, + "tokens_seen": 444792832 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043697091273821467, + "loss": 2.9083, + "theoretical_loss": 3.964942033746216, + "tokens_seen": 444858368 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369608826479438, + "loss": 2.8564, + "theoretical_loss": 3.9648778540196945, + "tokens_seen": 444923904 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043695085255767303, + "loss": 2.7252, + "theoretical_loss": 3.964813686392496, + "tokens_seen": 444989440 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369408224674022, + "loss": 2.992, + "theoretical_loss": 3.9647495308605594, + "tokens_seen": 445054976 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369307923771314, + "loss": 2.9756, + "theoretical_loss": 3.964685387419823, + "tokens_seen": 445120512 + }, + { + "epoch": 0.13, + "learning_rate": 0.0004369207622868606, + "loss": 2.8825, + "theoretical_loss": 3.964621256066229, + "tokens_seen": 445186048 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043691073219658975, + "loss": 2.804, + "theoretical_loss": 3.96455713679572, + "tokens_seen": 445251584 + }, + { + "epoch": 0.13, + "learning_rate": 0.000436900702106319, + "loss": 2.609, + "theoretical_loss": 3.9644930296042418, + "tokens_seen": 445317120 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043689067201604817, + "loss": 2.6154, + "theoretical_loss": 3.964428934487742, + "tokens_seen": 445382656 + }, + { + "epoch": 0.13, + "learning_rate": 0.00043688064192577735, + "loss": 2.8911, + "theoretical_loss": 3.96436485144217, + "tokens_seen": 445448192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043687061183550653, + "loss": 2.9299, + "theoretical_loss": 3.964300780463477, + "tokens_seen": 445513728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043686058174523577, + "loss": 2.6714, + "theoretical_loss": 3.964236721547616, + "tokens_seen": 445579264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 257279, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.354438304901123, + "objective/train/theoretical_loss": 3.964172674690542, + "objective/train/tokens_used": 466104800, + "theoretical_loss": 3.964172674690542, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368505516549649, + "loss": 2.6825, + "theoretical_loss": 3.964172674690542, + "tokens_seen": 445644800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043684052156469413, + "loss": 2.8716, + "theoretical_loss": 3.9641086398882126, + "tokens_seen": 445710336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043683049147442326, + "loss": 2.9858, + "theoretical_loss": 3.9640446171365866, + "tokens_seen": 445775872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004368204613841525, + "loss": 2.7256, + "theoretical_loss": 3.9639806064316256, + "tokens_seen": 445841408 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043681043129388167, + "loss": 2.6819, + "theoretical_loss": 3.963916607769292, + "tokens_seen": 445906944 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043680040120361085, + "loss": 3.0643, + "theoretical_loss": 3.963852621145551, + "tokens_seen": 445972480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043679037111334003, + "loss": 2.9543, + "theoretical_loss": 3.9637886465563685, + "tokens_seen": 446038016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367803410230692, + "loss": 3.0419, + "theoretical_loss": 3.9637246839977154, + "tokens_seen": 446103552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367703109327984, + "loss": 3.1, + "theoretical_loss": 3.9636607334655602, + "tokens_seen": 446169088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043676028084252763, + "loss": 2.8148, + "theoretical_loss": 3.9635967949558775, + "tokens_seen": 446234624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043675025075225676, + "loss": 2.8659, + "theoretical_loss": 3.963532868464641, + "tokens_seen": 446300160 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436740220661986, + "loss": 2.8417, + "theoretical_loss": 3.963468953987827, + "tokens_seen": 446365696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367301905717151, + "loss": 2.9253, + "theoretical_loss": 3.9634050515214145, + "tokens_seen": 446431232 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043672016048144436, + "loss": 3.0207, + "theoretical_loss": 3.963341161061384, + "tokens_seen": 446496768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043671013039117354, + "loss": 2.7421, + "theoretical_loss": 3.9632772826037175, + "tokens_seen": 446562304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004367001003009027, + "loss": 2.7032, + "theoretical_loss": 3.9632134161444, + "tokens_seen": 446627840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366900702106319, + "loss": 2.8559, + "theoretical_loss": 3.9631495616794172, + "tokens_seen": 446693376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043668004012036113, + "loss": 2.7473, + "theoretical_loss": 3.9630857192047566, + "tokens_seen": 446758912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043667001003009026, + "loss": 2.8741, + "theoretical_loss": 3.96302188871641, + "tokens_seen": 446824448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366599799398195, + "loss": 2.9051, + "theoretical_loss": 3.9629580702103686, + "tokens_seen": 446889984 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366499498495486, + "loss": 2.9917, + "theoretical_loss": 3.9628942636826263, + "tokens_seen": 446955520 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043663991975927786, + "loss": 2.7779, + "theoretical_loss": 3.9628304691291785, + "tokens_seen": 447021056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043662988966900704, + "loss": 3.0163, + "theoretical_loss": 3.962766686546024, + "tokens_seen": 447086592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366198595787362, + "loss": 2.7074, + "theoretical_loss": 3.962702915929162, + "tokens_seen": 447152128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004366098294884654, + "loss": 2.8268, + "theoretical_loss": 3.9626391572745945, + "tokens_seen": 447217664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 258449, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.09433913230896, + "objective/train/theoretical_loss": 3.9625754105783244, + "objective/train/tokens_used": 467743200, + "theoretical_loss": 3.9625754105783244, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365997993981946, + "loss": 2.9999, + "theoretical_loss": 3.9625754105783244, + "tokens_seen": 447283200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043658976930792376, + "loss": 2.6759, + "theoretical_loss": 3.9625116758363577, + "tokens_seen": 447348736 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436579739217653, + "loss": 3.012, + "theoretical_loss": 3.9624479530447023, + "tokens_seen": 447414272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043656970912738213, + "loss": 2.8363, + "theoretical_loss": 3.962384242199367, + "tokens_seen": 447479808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043655967903711136, + "loss": 2.981, + "theoretical_loss": 3.962320543296363, + "tokens_seen": 447545344 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365496489468405, + "loss": 2.9546, + "theoretical_loss": 3.9622568563317038, + "tokens_seen": 447610880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365396188565697, + "loss": 2.7905, + "theoretical_loss": 3.962193181301404, + "tokens_seen": 447676416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365295887662989, + "loss": 2.9201, + "theoretical_loss": 3.9621295182014813, + "tokens_seen": 447741952 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004365195586760281, + "loss": 2.8073, + "theoretical_loss": 3.9620658670279543, + "tokens_seen": 447807488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043650952858575727, + "loss": 3.0315, + "theoretical_loss": 3.9620022277768436, + "tokens_seen": 447873024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364994984954865, + "loss": 2.869, + "theoretical_loss": 3.9619386004441726, + "tokens_seen": 447938560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043648946840521563, + "loss": 2.9296, + "theoretical_loss": 3.9618749850259647, + "tokens_seen": 448004096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043647943831494487, + "loss": 3.0547, + "theoretical_loss": 3.961811381518248, + "tokens_seen": 448069632 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436469408224674, + "loss": 3.042, + "theoretical_loss": 3.96174778991705, + "tokens_seen": 448135168 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043645937813440323, + "loss": 2.8965, + "theoretical_loss": 3.961684210218401, + "tokens_seen": 448200704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364493480441324, + "loss": 2.8506, + "theoretical_loss": 3.9616206424183336, + "tokens_seen": 448266240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004364393179538616, + "loss": 2.8845, + "theoretical_loss": 3.961557086512882, + "tokens_seen": 448331776 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043642928786359077, + "loss": 2.824, + "theoretical_loss": 3.961493542498083, + "tokens_seen": 448397312 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043641925777331995, + "loss": 2.9255, + "theoretical_loss": 3.9614300103699733, + "tokens_seen": 448462848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043640922768304913, + "loss": 2.9331, + "theoretical_loss": 3.961366490124593, + "tokens_seen": 448528384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043639919759277837, + "loss": 3.0854, + "theoretical_loss": 3.961302981757984, + "tokens_seen": 448593920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363891675025075, + "loss": 2.6079, + "theoretical_loss": 3.9612394852661907, + "tokens_seen": 448659456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043637913741223673, + "loss": 2.8454, + "theoretical_loss": 3.961176000645258, + "tokens_seen": 448724992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363691073219659, + "loss": 3.0051, + "theoretical_loss": 3.9611125278912334, + "tokens_seen": 448790528 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363590772316951, + "loss": 2.703, + "theoretical_loss": 3.9610490670001663, + "tokens_seen": 448856064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 259697, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1037912368774414, + "objective/train/theoretical_loss": 3.9609856179681078, + "objective/train/tokens_used": 469381600, + "theoretical_loss": 3.9609856179681078, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004363490471414243, + "loss": 3.0174, + "theoretical_loss": 3.9609856179681078, + "tokens_seen": 448921600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043633901705115346, + "loss": 2.9102, + "theoretical_loss": 3.9609221807911115, + "tokens_seen": 448987136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043632898696088264, + "loss": 2.832, + "theoretical_loss": 3.960858755465232, + "tokens_seen": 449052672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043631895687061187, + "loss": 3.0512, + "theoretical_loss": 3.9607953419865263, + "tokens_seen": 449118208 + }, + { + "epoch": 0.14, + "learning_rate": 0.000436308926780341, + "loss": 2.8357, + "theoretical_loss": 3.960731940351053, + "tokens_seen": 449183744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043629889669007023, + "loss": 2.8914, + "theoretical_loss": 3.9606685505548738, + "tokens_seen": 449249280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043628886659979936, + "loss": 2.6328, + "theoretical_loss": 3.96060517259405, + "tokens_seen": 449314816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362788365095286, + "loss": 2.7281, + "theoretical_loss": 3.9605418064646467, + "tokens_seen": 449380352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362688064192578, + "loss": 3.0245, + "theoretical_loss": 3.9604784521627296, + "tokens_seen": 449445888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043625877632898696, + "loss": 3.0669, + "theoretical_loss": 3.9604151096843676, + "tokens_seen": 449511424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043624874623871614, + "loss": 2.9458, + "theoretical_loss": 3.9603517790256313, + "tokens_seen": 449576960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362387161484453, + "loss": 2.9036, + "theoretical_loss": 3.960288460182591, + "tokens_seen": 449642496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004362286860581745, + "loss": 3.0075, + "theoretical_loss": 3.960225153151322, + "tokens_seen": 449708032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043621865596790374, + "loss": 2.8153, + "theoretical_loss": 3.9601618579279, + "tokens_seen": 449773568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043620862587763286, + "loss": 2.9289, + "theoretical_loss": 3.960098574508402, + "tokens_seen": 449839104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361985957873621, + "loss": 3.0163, + "theoretical_loss": 3.9600353028889073, + "tokens_seen": 449904640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361885656970913, + "loss": 3.0982, + "theoretical_loss": 3.9599720430654983, + "tokens_seen": 449970176 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043617853560682046, + "loss": 2.7447, + "theoretical_loss": 3.9599087950342566, + "tokens_seen": 450035712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361685055165497, + "loss": 2.9742, + "theoretical_loss": 3.959845558791269, + "tokens_seen": 450101248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361584754262788, + "loss": 3.0095, + "theoretical_loss": 3.9597823343326213, + "tokens_seen": 450166784 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043614844533600806, + "loss": 2.6936, + "theoretical_loss": 3.9597191216544028, + "tokens_seen": 450232320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043613841524573724, + "loss": 2.9194, + "theoretical_loss": 3.959655920752704, + "tokens_seen": 450297856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361283851554664, + "loss": 2.8928, + "theoretical_loss": 3.959592731623618, + "tokens_seen": 450363392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361183550651956, + "loss": 2.9382, + "theoretical_loss": 3.9595295542632387, + "tokens_seen": 450428928 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004361083249749248, + "loss": 2.8739, + "theoretical_loss": 3.9594663886676624, + "tokens_seen": 450494464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 260246, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8196277618408203, + "objective/train/theoretical_loss": 3.9594032348329877, + "objective/train/tokens_used": 471020000, + "theoretical_loss": 3.9594032348329877, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043609829488465397, + "loss": 2.8036, + "theoretical_loss": 3.9594032348329877, + "tokens_seen": 450560000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360882647943832, + "loss": 2.8566, + "theoretical_loss": 3.959340092755314, + "tokens_seen": 450625536 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043607823470411233, + "loss": 2.8465, + "theoretical_loss": 3.959276962430744, + "tokens_seen": 450691072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043606820461384156, + "loss": 2.9252, + "theoretical_loss": 3.9592138438553808, + "tokens_seen": 450756608 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360581745235707, + "loss": 2.6681, + "theoretical_loss": 3.95915073702533, + "tokens_seen": 450822144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360481444332999, + "loss": 2.8351, + "theoretical_loss": 3.959087641936699, + "tokens_seen": 450887680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360381143430291, + "loss": 2.6263, + "theoretical_loss": 3.9590245585855977, + "tokens_seen": 450953216 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360280842527583, + "loss": 3.0796, + "theoretical_loss": 3.9589614869681364, + "tokens_seen": 451018752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043601805416248747, + "loss": 2.8513, + "theoretical_loss": 3.9588984270804293, + "tokens_seen": 451084288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004360080240722167, + "loss": 2.8436, + "theoretical_loss": 3.9588353789185895, + "tokens_seen": 451149824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043599799398194583, + "loss": 2.9357, + "theoretical_loss": 3.9587723424787358, + "tokens_seen": 451215360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043598796389167507, + "loss": 2.5351, + "theoretical_loss": 3.9587093177569854, + "tokens_seen": 451280896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359779338014042, + "loss": 3.0153, + "theoretical_loss": 3.958646304749459, + "tokens_seen": 451346432 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043596790371113343, + "loss": 2.7466, + "theoretical_loss": 3.9585833034522793, + "tokens_seen": 451411968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359578736208626, + "loss": 2.8312, + "theoretical_loss": 3.9585203138615697, + "tokens_seen": 451477504 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004359478435305918, + "loss": 2.7963, + "theoretical_loss": 3.9584573359734563, + "tokens_seen": 451543040 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043593781344032097, + "loss": 2.7758, + "theoretical_loss": 3.958394369784067, + "tokens_seen": 451608576 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043592778335005015, + "loss": 2.8486, + "theoretical_loss": 3.9583314152895324, + "tokens_seen": 451674112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043591775325977933, + "loss": 2.956, + "theoretical_loss": 3.9582684724859822, + "tokens_seen": 451739648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043590772316950857, + "loss": 2.7166, + "theoretical_loss": 3.9582055413695514, + "tokens_seen": 451805184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358976930792377, + "loss": 3.0765, + "theoretical_loss": 3.9581426219363745, + "tokens_seen": 451870720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043588766298896693, + "loss": 2.679, + "theoretical_loss": 3.958079714182588, + "tokens_seen": 451936256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358776328986961, + "loss": 2.9488, + "theoretical_loss": 3.958016818104331, + "tokens_seen": 452001792 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358676028084253, + "loss": 2.8143, + "theoretical_loss": 3.957953933697745, + "tokens_seen": 452067328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358575727181545, + "loss": 3.0067, + "theoretical_loss": 3.9578910609589713, + "tokens_seen": 452132864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 261277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.868929147720337, + "objective/train/theoretical_loss": 3.957828199884155, + "objective/train/tokens_used": 472658400, + "theoretical_loss": 3.957828199884155, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043584754262788366, + "loss": 2.8863, + "theoretical_loss": 3.957828199884155, + "tokens_seen": 452198400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043583751253761284, + "loss": 2.8311, + "theoretical_loss": 3.9577653504694426, + "tokens_seen": 452263936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043582748244734207, + "loss": 2.8063, + "theoretical_loss": 3.9577025127109815, + "tokens_seen": 452329472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004358174523570712, + "loss": 2.7761, + "theoretical_loss": 3.957639686604921, + "tokens_seen": 452395008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043580742226680043, + "loss": 2.8938, + "theoretical_loss": 3.9575768721474143, + "tokens_seen": 452460544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043579739217652956, + "loss": 3.0099, + "theoretical_loss": 3.9575140693346134, + "tokens_seen": 452526080 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357873620862588, + "loss": 2.7552, + "theoretical_loss": 3.957451278162674, + "tokens_seen": 452591616 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435777331995988, + "loss": 2.6856, + "theoretical_loss": 3.957388498627754, + "tokens_seen": 452657152 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043576730190571716, + "loss": 3.014, + "theoretical_loss": 3.957325730726012, + "tokens_seen": 452722688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043575727181544634, + "loss": 2.7964, + "theoretical_loss": 3.9572629744536085, + "tokens_seen": 452788224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357472417251755, + "loss": 2.9826, + "theoretical_loss": 3.9572002298067064, + "tokens_seen": 452853760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357372116349047, + "loss": 2.7544, + "theoretical_loss": 3.95713749678147, + "tokens_seen": 452919296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043572718154463394, + "loss": 2.8092, + "theoretical_loss": 3.957074775374066, + "tokens_seen": 452984832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043571715145436306, + "loss": 2.7585, + "theoretical_loss": 3.9570120655806615, + "tokens_seen": 453050368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004357071213640923, + "loss": 2.8345, + "theoretical_loss": 3.956949367397427, + "tokens_seen": 453115904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356970912738215, + "loss": 2.9775, + "theoretical_loss": 3.956886680820535, + "tokens_seen": 453181440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043568706118355066, + "loss": 2.9825, + "theoretical_loss": 3.9568240058461575, + "tokens_seen": 453246976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043567703109327984, + "loss": 2.964, + "theoretical_loss": 3.956761342470471, + "tokens_seen": 453312512 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435667001003009, + "loss": 2.8124, + "theoretical_loss": 3.9566986906896515, + "tokens_seen": 453378048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356569709127382, + "loss": 3.0178, + "theoretical_loss": 3.95663605049988, + "tokens_seen": 453443584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043564694082246744, + "loss": 2.9025, + "theoretical_loss": 3.9565734218973354, + "tokens_seen": 453509120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043563691073219657, + "loss": 2.8165, + "theoretical_loss": 3.9565108048782007, + "tokens_seen": 453574656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004356268806419258, + "loss": 3.0191, + "theoretical_loss": 3.9564481994386607, + "tokens_seen": 453640192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043561685055165493, + "loss": 2.9861, + "theoretical_loss": 3.9563856055749014, + "tokens_seen": 453705728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043560682046138417, + "loss": 2.9445, + "theoretical_loss": 3.9563230232831113, + "tokens_seen": 453771264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 261864, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.966676712036133, + "objective/train/theoretical_loss": 3.956260452559479, + "objective/train/tokens_used": 474296800, + "theoretical_loss": 3.956260452559479, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043559679037111335, + "loss": 2.7952, + "theoretical_loss": 3.956260452559479, + "tokens_seen": 453836800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043558676028084253, + "loss": 3.0193, + "theoretical_loss": 3.956197893400198, + "tokens_seen": 453902336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355767301905717, + "loss": 3.02, + "theoretical_loss": 3.95613534580146, + "tokens_seen": 453967872 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355667001003009, + "loss": 2.995, + "theoretical_loss": 3.956072809759461, + "tokens_seen": 454033408 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043555667001003007, + "loss": 2.8422, + "theoretical_loss": 3.9560102852703976, + "tokens_seen": 454098944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004355466399197593, + "loss": 2.8431, + "theoretical_loss": 3.955947772330469, + "tokens_seen": 454164480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043553660982948843, + "loss": 2.8094, + "theoretical_loss": 3.9558852709358763, + "tokens_seen": 454230016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043552657973921767, + "loss": 2.8377, + "theoretical_loss": 3.9558227810828215, + "tokens_seen": 454295552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043551654964894685, + "loss": 2.6956, + "theoretical_loss": 3.9557603027675086, + "tokens_seen": 454361088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043550651955867603, + "loss": 2.7347, + "theoretical_loss": 3.955697835986143, + "tokens_seen": 454426624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354964894684052, + "loss": 2.8711, + "theoretical_loss": 3.9556353807349343, + "tokens_seen": 454492160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354864593781344, + "loss": 2.9731, + "theoretical_loss": 3.955572937010091, + "tokens_seen": 454557696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354764292878636, + "loss": 2.93, + "theoretical_loss": 3.9555105048078243, + "tokens_seen": 454623232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354663991975928, + "loss": 2.7183, + "theoretical_loss": 3.9554480841243476, + "tokens_seen": 454688768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043545636910732194, + "loss": 2.9553, + "theoretical_loss": 3.9553856749558762, + "tokens_seen": 454754304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043544633901705117, + "loss": 2.8743, + "theoretical_loss": 3.9553232772986266, + "tokens_seen": 454819840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354363089267803, + "loss": 2.999, + "theoretical_loss": 3.955260891148818, + "tokens_seen": 454885376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043542627883650953, + "loss": 2.8236, + "theoretical_loss": 3.9551985165026693, + "tokens_seen": 454950912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043541624874623877, + "loss": 3.0874, + "theoretical_loss": 3.955136153356404, + "tokens_seen": 455016448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004354062186559679, + "loss": 2.794, + "theoretical_loss": 3.955073801706246, + "tokens_seen": 455081984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043539618856569713, + "loss": 2.8035, + "theoretical_loss": 3.9550114615484198, + "tokens_seen": 455147520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353861584754263, + "loss": 2.6472, + "theoretical_loss": 3.954949132879154, + "tokens_seen": 455213056 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353761283851555, + "loss": 2.7814, + "theoretical_loss": 3.9548868156946777, + "tokens_seen": 455278592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353660982948847, + "loss": 2.9024, + "theoretical_loss": 3.9548245099912216, + "tokens_seen": 455344128 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043535606820461386, + "loss": 2.7629, + "theoretical_loss": 3.954762215765019, + "tokens_seen": 455409664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 263055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.755061388015747, + "objective/train/theoretical_loss": 3.9546999330123036, + "objective/train/tokens_used": 475935200, + "theoretical_loss": 3.9546999330123036, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043534603811434304, + "loss": 2.8091, + "theoretical_loss": 3.9546999330123036, + "tokens_seen": 455475200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043533600802407227, + "loss": 2.7972, + "theoretical_loss": 3.954637661729312, + "tokens_seen": 455540736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004353259779338014, + "loss": 2.7967, + "theoretical_loss": 3.954575401912284, + "tokens_seen": 455606272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043531594784353063, + "loss": 2.8888, + "theoretical_loss": 3.954513153557458, + "tokens_seen": 455671808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043530591775325976, + "loss": 2.7848, + "theoretical_loss": 3.9544509166610755, + "tokens_seen": 455737344 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435295887662989, + "loss": 2.7125, + "theoretical_loss": 3.95438869121938, + "tokens_seen": 455802880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352858575727182, + "loss": 2.8886, + "theoretical_loss": 3.9543264772286184, + "tokens_seen": 455868416 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043527582748244736, + "loss": 2.7964, + "theoretical_loss": 3.954264274685036, + "tokens_seen": 455933952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043526579739217654, + "loss": 3.0038, + "theoretical_loss": 3.954202083584882, + "tokens_seen": 455999488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352557673019057, + "loss": 3.0308, + "theoretical_loss": 3.954139903924407, + "tokens_seen": 456065024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352457372116349, + "loss": 2.8824, + "theoretical_loss": 3.9540777356998635, + "tokens_seen": 456130560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043523570712136414, + "loss": 2.8996, + "theoretical_loss": 3.954015578907506, + "tokens_seen": 456196096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043522567703109326, + "loss": 2.8487, + "theoretical_loss": 3.953953433543589, + "tokens_seen": 456261632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352156469408225, + "loss": 2.7798, + "theoretical_loss": 3.953891299604371, + "tokens_seen": 456327168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004352056168505517, + "loss": 2.8852, + "theoretical_loss": 3.9538291770861123, + "tokens_seen": 456392704 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043519558676028086, + "loss": 3.0216, + "theoretical_loss": 3.9537670659850725, + "tokens_seen": 456458240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043518555667001004, + "loss": 2.8016, + "theoretical_loss": 3.953704966297515, + "tokens_seen": 456523776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351755265797392, + "loss": 2.7142, + "theoretical_loss": 3.953642878019705, + "tokens_seen": 456589312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004351654964894684, + "loss": 2.846, + "theoretical_loss": 3.953580801147908, + "tokens_seen": 456654848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043515546639919764, + "loss": 2.9923, + "theoretical_loss": 3.9535187356783927, + "tokens_seen": 456720384 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043514543630892677, + "loss": 2.7513, + "theoretical_loss": 3.9534566816074292, + "tokens_seen": 456785920 + }, + { + "epoch": 0.14, + "learning_rate": 0.000435135406218656, + "loss": 2.7321, + "theoretical_loss": 3.953394638931289, + "tokens_seen": 456851456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043512537612838513, + "loss": 2.8101, + "theoretical_loss": 3.953332607646246, + "tokens_seen": 456916992 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043511534603811437, + "loss": 2.754, + "theoretical_loss": 3.953270587748574, + "tokens_seen": 456982528 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043510531594784355, + "loss": 2.8689, + "theoretical_loss": 3.9532085792345515, + "tokens_seen": 457048064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 264378, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8818917274475098, + "objective/train/theoretical_loss": 3.953146582100456, + "objective/train/tokens_used": 477573600, + "theoretical_loss": 3.953146582100456, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043509528585757273, + "loss": 2.9812, + "theoretical_loss": 3.953146582100456, + "tokens_seen": 457113600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350852557673019, + "loss": 2.7279, + "theoretical_loss": 3.9530845963425696, + "tokens_seen": 457179136 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350752256770311, + "loss": 2.69, + "theoretical_loss": 3.9530226219571727, + "tokens_seen": 457244672 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043506519558676027, + "loss": 2.698, + "theoretical_loss": 3.9529606589405497, + "tokens_seen": 457310208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350551654964895, + "loss": 2.8551, + "theoretical_loss": 3.952898707288987, + "tokens_seen": 457375744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043504513540621863, + "loss": 2.8347, + "theoretical_loss": 3.9528367669987716, + "tokens_seen": 457441280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043503510531594787, + "loss": 2.828, + "theoretical_loss": 3.952774838066193, + "tokens_seen": 457506816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043502507522567705, + "loss": 2.8248, + "theoretical_loss": 3.9527129204875413, + "tokens_seen": 457572352 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043501504513540623, + "loss": 3.1028, + "theoretical_loss": 3.95265101425911, + "tokens_seen": 457637888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004350050150451354, + "loss": 3.2986, + "theoretical_loss": 3.952589119377193, + "tokens_seen": 457703424 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349949849548646, + "loss": 2.7862, + "theoretical_loss": 3.9525272358380867, + "tokens_seen": 457768960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349849548645938, + "loss": 2.91, + "theoretical_loss": 3.9524653636380895, + "tokens_seen": 457834496 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434974924774323, + "loss": 2.9778, + "theoretical_loss": 3.9524035027734996, + "tokens_seen": 457900032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043496489468405214, + "loss": 2.9951, + "theoretical_loss": 3.9523416532406204, + "tokens_seen": 457965568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043495486459378137, + "loss": 2.9673, + "theoretical_loss": 3.9522798150357525, + "tokens_seen": 458031104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349448345035105, + "loss": 2.7049, + "theoretical_loss": 3.952217988155203, + "tokens_seen": 458096640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043493480441323973, + "loss": 2.7651, + "theoretical_loss": 3.952156172595277, + "tokens_seen": 458162176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349247743229689, + "loss": 2.9111, + "theoretical_loss": 3.952094368352284, + "tokens_seen": 458227712 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349147442326981, + "loss": 3.0533, + "theoretical_loss": 3.9520325754225327, + "tokens_seen": 458293248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004349047141424273, + "loss": 2.8072, + "theoretical_loss": 3.951970793802336, + "tokens_seen": 458358784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348946840521565, + "loss": 2.4948, + "theoretical_loss": 3.9519090234880068, + "tokens_seen": 458424320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043488465396188564, + "loss": 2.8712, + "theoretical_loss": 3.9518472644758607, + "tokens_seen": 458489856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348746238716149, + "loss": 2.9302, + "theoretical_loss": 3.951785516762214, + "tokens_seen": 458555392 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434864593781344, + "loss": 2.7907, + "theoretical_loss": 3.951723780343386, + "tokens_seen": 458620928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043485456369107324, + "loss": 2.9902, + "theoretical_loss": 3.951662055215697, + "tokens_seen": 458686464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 264937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.118872880935669, + "objective/train/theoretical_loss": 3.951600341375469, + "objective/train/tokens_used": 479212000, + "theoretical_loss": 3.951600341375469, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348445336008024, + "loss": 3.0182, + "theoretical_loss": 3.951600341375469, + "tokens_seen": 458752000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348345035105316, + "loss": 2.8335, + "theoretical_loss": 3.9515386388190255, + "tokens_seen": 458817536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004348244734202608, + "loss": 2.8455, + "theoretical_loss": 3.9514769475426927, + "tokens_seen": 458883072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043481444332998996, + "loss": 2.9251, + "theoretical_loss": 3.9514152675427976, + "tokens_seen": 458948608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043480441323971914, + "loss": 2.7918, + "theoretical_loss": 3.9513535988156696, + "tokens_seen": 459014144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347943831494484, + "loss": 2.7582, + "theoretical_loss": 3.9512919413576384, + "tokens_seen": 459079680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347843530591775, + "loss": 2.9296, + "theoretical_loss": 3.951230295165038, + "tokens_seen": 459145216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043477432296890674, + "loss": 2.8319, + "theoretical_loss": 3.951168660234201, + "tokens_seen": 459210752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043476429287863587, + "loss": 2.6616, + "theoretical_loss": 3.9511070365614644, + "tokens_seen": 459276288 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347542627883651, + "loss": 2.8487, + "theoretical_loss": 3.951045424143165, + "tokens_seen": 459341824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347442326980943, + "loss": 2.7885, + "theoretical_loss": 3.9509838229756427, + "tokens_seen": 459407360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043473420260782346, + "loss": 2.9401, + "theoretical_loss": 3.950922233055238, + "tokens_seen": 459472896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043472417251755265, + "loss": 2.8083, + "theoretical_loss": 3.9508606543782943, + "tokens_seen": 459538432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004347141424272819, + "loss": 2.8683, + "theoretical_loss": 3.9507990869411556, + "tokens_seen": 459603968 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434704112337011, + "loss": 2.7853, + "theoretical_loss": 3.9507375307401684, + "tokens_seen": 459669504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043469408224674024, + "loss": 3.0062, + "theoretical_loss": 3.9506759857716798, + "tokens_seen": 459735040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346840521564694, + "loss": 2.7083, + "theoretical_loss": 3.95061445203204, + "tokens_seen": 459800576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346740220661986, + "loss": 3.0644, + "theoretical_loss": 3.9505529295176, + "tokens_seen": 459866112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043466399197592784, + "loss": 2.6074, + "theoretical_loss": 3.950491418224713, + "tokens_seen": 459931648 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043465396188565697, + "loss": 2.8359, + "theoretical_loss": 3.9504299181497338, + "tokens_seen": 459997184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004346439317953862, + "loss": 2.6802, + "theoretical_loss": 3.9503684292890187, + "tokens_seen": 460062720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043463390170511533, + "loss": 3.035, + "theoretical_loss": 3.9503069516389253, + "tokens_seen": 460128256 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043462387161484457, + "loss": 2.9591, + "theoretical_loss": 3.950245485195814, + "tokens_seen": 460193792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043461384152457375, + "loss": 2.8343, + "theoretical_loss": 3.9501840299560467, + "tokens_seen": 460259328 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043460381143430293, + "loss": 2.5444, + "theoretical_loss": 3.950122585915985, + "tokens_seen": 460324864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 265992, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.820603370666504, + "objective/train/theoretical_loss": 3.9500611530719953, + "objective/train/tokens_used": 480850400, + "theoretical_loss": 3.9500611530719953, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345937813440321, + "loss": 2.7728, + "theoretical_loss": 3.9500611530719953, + "tokens_seen": 460390400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345837512537613, + "loss": 2.894, + "theoretical_loss": 3.9499997314204434, + "tokens_seen": 460455936 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043457372116349047, + "loss": 2.8096, + "theoretical_loss": 3.949938320957698, + "tokens_seen": 460521472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345636910732197, + "loss": 2.4647, + "theoretical_loss": 3.949876921680129, + "tokens_seen": 460587008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043455366098294883, + "loss": 3.0128, + "theoretical_loss": 3.9498155335841076, + "tokens_seen": 460652544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043454363089267807, + "loss": 2.9574, + "theoretical_loss": 3.949754156666008, + "tokens_seen": 460718080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043453360080240725, + "loss": 2.8107, + "theoretical_loss": 3.949692790922205, + "tokens_seen": 460783616 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043452357071213643, + "loss": 2.7675, + "theoretical_loss": 3.9496314363490743, + "tokens_seen": 460849152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345135406218656, + "loss": 2.7283, + "theoretical_loss": 3.949570092942996, + "tokens_seen": 460914688 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004345035105315948, + "loss": 2.7483, + "theoretical_loss": 3.949508760700349, + "tokens_seen": 460980224 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434493480441324, + "loss": 2.7427, + "theoretical_loss": 3.949447439617516, + "tokens_seen": 461045760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344834503510532, + "loss": 2.9061, + "theoretical_loss": 3.94938612969088, + "tokens_seen": 461111296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043447342026078234, + "loss": 2.9127, + "theoretical_loss": 3.949324830916826, + "tokens_seen": 461176832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043446339017051157, + "loss": 2.9454, + "theoretical_loss": 3.949263543291741, + "tokens_seen": 461242368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344533600802407, + "loss": 2.848, + "theoretical_loss": 3.949202266812014, + "tokens_seen": 461307904 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043444332998996993, + "loss": 3.0778, + "theoretical_loss": 3.9491410014740347, + "tokens_seen": 461373440 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344332998996991, + "loss": 2.9834, + "theoretical_loss": 3.949079747274195, + "tokens_seen": 461438976 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344232698094283, + "loss": 2.8894, + "theoretical_loss": 3.9490185042088894, + "tokens_seen": 461504512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344132397191575, + "loss": 2.653, + "theoretical_loss": 3.948957272274512, + "tokens_seen": 461570048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004344032096288867, + "loss": 2.8366, + "theoretical_loss": 3.94889605146746, + "tokens_seen": 461635584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043439317953861584, + "loss": 2.9733, + "theoretical_loss": 3.948834841784133, + "tokens_seen": 461701120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343831494483451, + "loss": 2.7639, + "theoretical_loss": 3.9487736432209304, + "tokens_seen": 461766656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343731193580742, + "loss": 3.0074, + "theoretical_loss": 3.9487124557742543, + "tokens_seen": 461832192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043436308926780344, + "loss": 3.0052, + "theoretical_loss": 3.948651279440509, + "tokens_seen": 461897728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343530591775326, + "loss": 3.129, + "theoretical_loss": 3.948590114216098, + "tokens_seen": 461963264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 266575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.696946144104004, + "objective/train/theoretical_loss": 3.9485289600974305, + "objective/train/tokens_used": 482488800, + "theoretical_loss": 3.9485289600974305, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343430290872618, + "loss": 2.6741, + "theoretical_loss": 3.9485289600974305, + "tokens_seen": 462028800 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434332998996991, + "loss": 2.8477, + "theoretical_loss": 3.948467817080915, + "tokens_seen": 462094336 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043432296890672016, + "loss": 3.0804, + "theoretical_loss": 3.94840668516296, + "tokens_seen": 462159872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043431293881644934, + "loss": 2.9808, + "theoretical_loss": 3.9483455643399794, + "tokens_seen": 462225408 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004343029087261786, + "loss": 3.0524, + "theoretical_loss": 3.9482844546083857, + "tokens_seen": 462290944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342928786359077, + "loss": 2.8848, + "theoretical_loss": 3.948223355964595, + "tokens_seen": 462356480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043428284854563694, + "loss": 2.9191, + "theoretical_loss": 3.948162268405024, + "tokens_seen": 462422016 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043427281845536607, + "loss": 2.7022, + "theoretical_loss": 3.9481011919260913, + "tokens_seen": 462487552 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342627883650953, + "loss": 2.759, + "theoretical_loss": 3.9480401265242175, + "tokens_seen": 462553088 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342527582748245, + "loss": 2.7993, + "theoretical_loss": 3.9479790721958246, + "tokens_seen": 462618624 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043424272818455367, + "loss": 2.7651, + "theoretical_loss": 3.9479180289373366, + "tokens_seen": 462684160 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043423269809428285, + "loss": 2.8418, + "theoretical_loss": 3.9478569967451778, + "tokens_seen": 462749696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342226680040121, + "loss": 2.7632, + "theoretical_loss": 3.947795975615776, + "tokens_seen": 462815232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004342126379137412, + "loss": 2.9218, + "theoretical_loss": 3.9477349655455596, + "tokens_seen": 462880768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043420260782347044, + "loss": 2.9163, + "theoretical_loss": 3.9476739665309593, + "tokens_seen": 462946304 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043419257773319957, + "loss": 2.6965, + "theoretical_loss": 3.947612978568406, + "tokens_seen": 463011840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341825476429288, + "loss": 2.9593, + "theoretical_loss": 3.9475520016543353, + "tokens_seen": 463077376 + }, + { + "epoch": 0.14, + "learning_rate": 0.000434172517552658, + "loss": 2.9816, + "theoretical_loss": 3.9474910357851805, + "tokens_seen": 463142912 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043416248746238717, + "loss": 2.9265, + "theoretical_loss": 3.9474300809573797, + "tokens_seen": 463208448 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043415245737211635, + "loss": 2.7887, + "theoretical_loss": 3.947369137167371, + "tokens_seen": 463273984 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043414242728184553, + "loss": 2.7265, + "theoretical_loss": 3.947308204411595, + "tokens_seen": 463339520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341323971915747, + "loss": 2.8877, + "theoretical_loss": 3.947247282686493, + "tokens_seen": 463405056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043412236710130395, + "loss": 2.8725, + "theoretical_loss": 3.947186371988509, + "tokens_seen": 463470592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341123370110331, + "loss": 2.7452, + "theoretical_loss": 3.947125472314088, + "tokens_seen": 463536128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004341023069207623, + "loss": 2.7315, + "theoretical_loss": 3.947064583659678, + "tokens_seen": 463601664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 267814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8467938899993896, + "objective/train/theoretical_loss": 3.9470037060217256, + "objective/train/tokens_used": 484127200, + "theoretical_loss": 3.9470037060217256, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043409227683049144, + "loss": 2.9119, + "theoretical_loss": 3.9470037060217256, + "tokens_seen": 463667200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043408224674022067, + "loss": 2.7476, + "theoretical_loss": 3.946942839396682, + "tokens_seen": 463732736 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043407221664994985, + "loss": 2.8435, + "theoretical_loss": 3.946881983780999, + "tokens_seen": 463798272 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043406218655967903, + "loss": 3.0763, + "theoretical_loss": 3.9468211391711296, + "tokens_seen": 463863808 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340521564694082, + "loss": 2.8394, + "theoretical_loss": 3.9467603055635294, + "tokens_seen": 463929344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043404212637913745, + "loss": 2.9131, + "theoretical_loss": 3.9466994829546542, + "tokens_seen": 463994880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340320962888666, + "loss": 3.0057, + "theoretical_loss": 3.946638671340964, + "tokens_seen": 464060416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340220661985958, + "loss": 2.8754, + "theoretical_loss": 3.946577870718917, + "tokens_seen": 464125952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043401203610832494, + "loss": 3.0896, + "theoretical_loss": 3.9465170810849757, + "tokens_seen": 464191488 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004340020060180542, + "loss": 2.7866, + "theoretical_loss": 3.946456302435603, + "tokens_seen": 464257024 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043399197592778336, + "loss": 2.8963, + "theoretical_loss": 3.946395534767265, + "tokens_seen": 464322560 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043398194583751254, + "loss": 2.9696, + "theoretical_loss": 3.946334778076427, + "tokens_seen": 464388096 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339719157472417, + "loss": 2.8131, + "theoretical_loss": 3.9462740323595575, + "tokens_seen": 464453632 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339618856569709, + "loss": 2.8574, + "theoretical_loss": 3.9462132976131263, + "tokens_seen": 464519168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339518555667001, + "loss": 2.7606, + "theoretical_loss": 3.946152573833605, + "tokens_seen": 464584704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339418254764293, + "loss": 3.0352, + "theoretical_loss": 3.9460918610174662, + "tokens_seen": 464650240 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339317953861585, + "loss": 2.7562, + "theoretical_loss": 3.946031159161185, + "tokens_seen": 464715776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339217652958877, + "loss": 2.8604, + "theoretical_loss": 3.9459704682612387, + "tokens_seen": 464781312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004339117352056169, + "loss": 2.8481, + "theoretical_loss": 3.945909788314104, + "tokens_seen": 464846848 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043390170511534604, + "loss": 2.7717, + "theoretical_loss": 3.945849119316261, + "tokens_seen": 464912384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338916750250753, + "loss": 2.7045, + "theoretical_loss": 3.9457884612641902, + "tokens_seen": 464977920 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338816449348044, + "loss": 2.9895, + "theoretical_loss": 3.9457278141543752, + "tokens_seen": 465043456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043387161484453364, + "loss": 2.8658, + "theoretical_loss": 3.9456671779833004, + "tokens_seen": 465108992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338615847542628, + "loss": 2.9028, + "theoretical_loss": 3.945606552747452, + "tokens_seen": 465174528 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433851554663992, + "loss": 2.954, + "theoretical_loss": 3.945545938443318, + "tokens_seen": 465240064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 268427, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.444042682647705, + "objective/train/theoretical_loss": 3.945485335067386, + "objective/train/tokens_used": 485765600, + "theoretical_loss": 3.945485335067386, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338415245737212, + "loss": 2.6741, + "theoretical_loss": 3.945485335067386, + "tokens_seen": 465305600 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043383149448345036, + "loss": 2.9785, + "theoretical_loss": 3.945424742616149, + "tokens_seen": 465371136 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043382146439317954, + "loss": 2.8609, + "theoretical_loss": 3.945364161086099, + "tokens_seen": 465436672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338114343029088, + "loss": 2.82, + "theoretical_loss": 3.9453035904737304, + "tokens_seen": 465502208 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004338014042126379, + "loss": 2.964, + "theoretical_loss": 3.9452430307755386, + "tokens_seen": 465567744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043379137412236714, + "loss": 2.9767, + "theoretical_loss": 3.945182481988021, + "tokens_seen": 465633280 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043378134403209627, + "loss": 2.921, + "theoretical_loss": 3.945121944107677, + "tokens_seen": 465698816 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337713139418255, + "loss": 2.8756, + "theoretical_loss": 3.945061417131007, + "tokens_seen": 465764352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337612838515547, + "loss": 2.721, + "theoretical_loss": 3.945000901054514, + "tokens_seen": 465829888 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043375125376128387, + "loss": 2.9334, + "theoretical_loss": 3.944940395874701, + "tokens_seen": 465895424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043374122367101305, + "loss": 2.9868, + "theoretical_loss": 3.944879901588074, + "tokens_seen": 465960960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337311935807423, + "loss": 2.7399, + "theoretical_loss": 3.9448194181911402, + "tokens_seen": 466026496 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004337211634904714, + "loss": 3.0384, + "theoretical_loss": 3.944758945680408, + "tokens_seen": 466092032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043371113340020064, + "loss": 2.8814, + "theoretical_loss": 3.9446984840523873, + "tokens_seen": 466157568 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043370110330992977, + "loss": 2.6462, + "theoretical_loss": 3.9446380333035918, + "tokens_seen": 466223104 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433691073219659, + "loss": 2.6692, + "theoretical_loss": 3.944577593430534, + "tokens_seen": 466288640 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336810431293882, + "loss": 2.8248, + "theoretical_loss": 3.9445171644297288, + "tokens_seen": 466354176 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043367101303911737, + "loss": 2.6765, + "theoretical_loss": 3.9444567462976936, + "tokens_seen": 466419712 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043366098294884655, + "loss": 2.8589, + "theoretical_loss": 3.9443963390309458, + "tokens_seen": 466485248 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043365095285857573, + "loss": 2.9641, + "theoretical_loss": 3.9443359426260063, + "tokens_seen": 466550784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336409227683049, + "loss": 2.8887, + "theoretical_loss": 3.944275557079397, + "tokens_seen": 466616320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043363089267803415, + "loss": 2.9296, + "theoretical_loss": 3.9442151823876404, + "tokens_seen": 466681856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336208625877633, + "loss": 2.7081, + "theoretical_loss": 3.9441548185472612, + "tokens_seen": 466747392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004336108324974925, + "loss": 2.7055, + "theoretical_loss": 3.9440944655547865, + "tokens_seen": 466812928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043360080240722164, + "loss": 3.0324, + "theoretical_loss": 3.944034123406744, + "tokens_seen": 466878464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 269429, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1784250736236572, + "objective/train/theoretical_loss": 3.943973792099663, + "objective/train/tokens_used": 487404000, + "theoretical_loss": 3.943973792099663, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043359077231695087, + "loss": 2.9881, + "theoretical_loss": 3.943973792099663, + "tokens_seen": 466944000 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043358074222668005, + "loss": 2.8874, + "theoretical_loss": 3.943913471630075, + "tokens_seen": 467009536 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043357071213640923, + "loss": 2.8577, + "theoretical_loss": 3.943853161994513, + "tokens_seen": 467075072 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335606820461384, + "loss": 2.9665, + "theoretical_loss": 3.943792863189511, + "tokens_seen": 467140608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043355065195586765, + "loss": 2.8564, + "theoretical_loss": 3.9437325752116053, + "tokens_seen": 467206144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335406218655968, + "loss": 2.8001, + "theoretical_loss": 3.943672298057333, + "tokens_seen": 467271680 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433530591775326, + "loss": 3.0148, + "theoretical_loss": 3.9436120317232337, + "tokens_seen": 467337216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043352056168505514, + "loss": 2.9508, + "theoretical_loss": 3.9435517762058483, + "tokens_seen": 467402752 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004335105315947844, + "loss": 2.9171, + "theoretical_loss": 3.9434915315017185, + "tokens_seen": 467468288 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043350050150451356, + "loss": 2.7347, + "theoretical_loss": 3.943431297607389, + "tokens_seen": 467533824 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043349047141424274, + "loss": 2.8175, + "theoretical_loss": 3.943371074519405, + "tokens_seen": 467599360 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334804413239719, + "loss": 2.5834, + "theoretical_loss": 3.943310862234313, + "tokens_seen": 467664896 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334704112337011, + "loss": 2.8371, + "theoretical_loss": 3.9432506607486633, + "tokens_seen": 467730432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334603811434303, + "loss": 2.6997, + "theoretical_loss": 3.9431904700590046, + "tokens_seen": 467795968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334503510531595, + "loss": 2.9174, + "theoretical_loss": 3.9431302901618897, + "tokens_seen": 467861504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043344032096288864, + "loss": 2.9733, + "theoretical_loss": 3.943070121053872, + "tokens_seen": 467927040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334302908726179, + "loss": 3.0873, + "theoretical_loss": 3.943009962731506, + "tokens_seen": 467992576 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433420260782347, + "loss": 2.8206, + "theoretical_loss": 3.9429498151913487, + "tokens_seen": 468058112 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043341023069207624, + "loss": 2.6554, + "theoretical_loss": 3.9428896784299585, + "tokens_seen": 468123648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004334002006018054, + "loss": 2.8164, + "theoretical_loss": 3.9428295524438948, + "tokens_seen": 468189184 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333901705115346, + "loss": 2.8438, + "theoretical_loss": 3.9427694372297193, + "tokens_seen": 468254720 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333801404212638, + "loss": 2.8599, + "theoretical_loss": 3.942709332783995, + "tokens_seen": 468320256 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433370110330993, + "loss": 2.7072, + "theoretical_loss": 3.942649239103286, + "tokens_seen": 468385792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043336008024072215, + "loss": 2.7251, + "theoretical_loss": 3.942589156184159, + "tokens_seen": 468451328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333500501504514, + "loss": 2.8781, + "theoretical_loss": 3.942529084023181, + "tokens_seen": 468516864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 269989, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7550222873687744, + "objective/train/theoretical_loss": 3.9424690226169212, + "objective/train/tokens_used": 489042400, + "theoretical_loss": 3.9424690226169212, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333400200601805, + "loss": 2.889, + "theoretical_loss": 3.9424690226169212, + "tokens_seen": 468582400 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043332998996990974, + "loss": 2.9478, + "theoretical_loss": 3.9424089719619513, + "tokens_seen": 468647936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333199598796389, + "loss": 2.7374, + "theoretical_loss": 3.942348932054843, + "tokens_seen": 468713472 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004333099297893681, + "loss": 2.8397, + "theoretical_loss": 3.942288902892171, + "tokens_seen": 468779008 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332998996990973, + "loss": 2.8605, + "theoretical_loss": 3.94222888447051, + "tokens_seen": 468844544 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043328986960882647, + "loss": 2.8271, + "theoretical_loss": 3.942168876786438, + "tokens_seen": 468910080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043327983951855565, + "loss": 2.7673, + "theoretical_loss": 3.942108879836532, + "tokens_seen": 468975616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332698094282849, + "loss": 2.8577, + "theoretical_loss": 3.942048893617374, + "tokens_seen": 469041152 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433259779338014, + "loss": 2.8374, + "theoretical_loss": 3.9419889181255456, + "tokens_seen": 469106688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043324974924774325, + "loss": 2.8189, + "theoretical_loss": 3.941928953357629, + "tokens_seen": 469172224 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043323971915747243, + "loss": 2.5255, + "theoretical_loss": 3.941868999310211, + "tokens_seen": 469237760 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332296890672016, + "loss": 2.6435, + "theoretical_loss": 3.9418090559798764, + "tokens_seen": 469303296 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004332196589769308, + "loss": 2.8742, + "theoretical_loss": 3.941749123363214, + "tokens_seen": 469368832 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043320962888665997, + "loss": 2.7471, + "theoretical_loss": 3.9416892014568132, + "tokens_seen": 469434368 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043319959879638915, + "loss": 2.8973, + "theoretical_loss": 3.941629290257266, + "tokens_seen": 469499904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331895687061184, + "loss": 2.7402, + "theoretical_loss": 3.941569389761164, + "tokens_seen": 469565440 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043317953861584757, + "loss": 2.7395, + "theoretical_loss": 3.941509499965102, + "tokens_seen": 469630976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043316950852557675, + "loss": 2.9798, + "theoretical_loss": 3.9414496208656766, + "tokens_seen": 469696512 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043315947843530593, + "loss": 2.8967, + "theoretical_loss": 3.941389752459484, + "tokens_seen": 469762048 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331494483450351, + "loss": 2.9411, + "theoretical_loss": 3.941329894743124, + "tokens_seen": 469827584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043313941825476435, + "loss": 2.9326, + "theoretical_loss": 3.9412700477131972, + "tokens_seen": 469893120 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331293881644935, + "loss": 2.8993, + "theoretical_loss": 3.9412102113663057, + "tokens_seen": 469958656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004331193580742227, + "loss": 2.8847, + "theoretical_loss": 3.941150385699052, + "tokens_seen": 470024192 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043310932798395184, + "loss": 2.7375, + "theoretical_loss": 3.941090570708043, + "tokens_seen": 470089728 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043309929789368107, + "loss": 3.0306, + "theoretical_loss": 3.9410307663898845, + "tokens_seen": 470155264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 271247, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.501755475997925, + "objective/train/theoretical_loss": 3.9409709727411846, + "objective/train/tokens_used": 490680800, + "theoretical_loss": 3.9409709727411846, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043308926780341025, + "loss": 2.8002, + "theoretical_loss": 3.9409709727411846, + "tokens_seen": 470220800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043307923771313943, + "loss": 2.8405, + "theoretical_loss": 3.940911189758554, + "tokens_seen": 470286336 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330692076228686, + "loss": 2.6275, + "theoretical_loss": 3.940851417438604, + "tokens_seen": 470351872 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043305917753259785, + "loss": 2.8477, + "theoretical_loss": 3.9407916557779465, + "tokens_seen": 470417408 + }, + { + "epoch": 0.14, + "learning_rate": 0.000433049147442327, + "loss": 2.8173, + "theoretical_loss": 3.9407319047731972, + "tokens_seen": 470482944 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330391173520562, + "loss": 2.7989, + "theoretical_loss": 3.940672164420972, + "tokens_seen": 470548480 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043302908726178534, + "loss": 2.9784, + "theoretical_loss": 3.940612434717888, + "tokens_seen": 470614016 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004330190571715146, + "loss": 3.0426, + "theoretical_loss": 3.9405527156605644, + "tokens_seen": 470679552 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043300902708124376, + "loss": 2.8404, + "theoretical_loss": 3.9404930072456223, + "tokens_seen": 470745088 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043299899699097294, + "loss": 2.6866, + "theoretical_loss": 3.9404333094696837, + "tokens_seen": 470810624 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329889669007021, + "loss": 2.8504, + "theoretical_loss": 3.940373622329372, + "tokens_seen": 470876160 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329789368104313, + "loss": 2.8516, + "theoretical_loss": 3.940313945821314, + "tokens_seen": 470941696 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329689067201605, + "loss": 2.9968, + "theoretical_loss": 3.9402542799421347, + "tokens_seen": 471007232 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329588766298897, + "loss": 2.8706, + "theoretical_loss": 3.9401946246884636, + "tokens_seen": 471072768 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043294884653961884, + "loss": 2.8943, + "theoretical_loss": 3.94013498005693, + "tokens_seen": 471138304 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329388164493481, + "loss": 2.789, + "theoretical_loss": 3.9400753460441655, + "tokens_seen": 471203840 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329287863590772, + "loss": 2.6528, + "theoretical_loss": 3.940015722646804, + "tokens_seen": 471269376 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043291875626880644, + "loss": 2.8134, + "theoretical_loss": 3.9399561098614786, + "tokens_seen": 471334912 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004329087261785356, + "loss": 3.0425, + "theoretical_loss": 3.939896507684827, + "tokens_seen": 471400448 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328986960882648, + "loss": 2.8502, + "theoretical_loss": 3.939836916113485, + "tokens_seen": 471465984 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432888665997994, + "loss": 2.7366, + "theoretical_loss": 3.939777335144094, + "tokens_seen": 471531520 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328786359077232, + "loss": 2.9817, + "theoretical_loss": 3.9397177647732926, + "tokens_seen": 471597056 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043286860581745235, + "loss": 2.9623, + "theoretical_loss": 3.9396582049977242, + "tokens_seen": 471662592 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328585757271816, + "loss": 2.7969, + "theoretical_loss": 3.9395986558140317, + "tokens_seen": 471728128 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328485456369107, + "loss": 2.6316, + "theoretical_loss": 3.939539117218861, + "tokens_seen": 471793664 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 272462, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.128762722015381, + "objective/train/theoretical_loss": 3.9394795892088585, + "objective/train/tokens_used": 492319200, + "theoretical_loss": 3.9394795892088585, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043283851554663994, + "loss": 2.9032, + "theoretical_loss": 3.9394795892088585, + "tokens_seen": 471859200 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328284854563691, + "loss": 2.7612, + "theoretical_loss": 3.939420071780673, + "tokens_seen": 471924736 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328184553660983, + "loss": 2.8278, + "theoretical_loss": 3.939360564930954, + "tokens_seen": 471990272 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004328084252758275, + "loss": 2.7656, + "theoretical_loss": 3.939301068656353, + "tokens_seen": 472055808 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043279839518555667, + "loss": 2.8529, + "theoretical_loss": 3.939241582953523, + "tokens_seen": 472121344 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043278836509528585, + "loss": 2.8641, + "theoretical_loss": 3.9391821078191183, + "tokens_seen": 472186880 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327783350050151, + "loss": 2.9301, + "theoretical_loss": 3.939122643249795, + "tokens_seen": 472252416 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327683049147442, + "loss": 2.794, + "theoretical_loss": 3.9390631892422103, + "tokens_seen": 472317952 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043275827482447345, + "loss": 2.9563, + "theoretical_loss": 3.9390037457930234, + "tokens_seen": 472383488 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043274824473420263, + "loss": 2.6417, + "theoretical_loss": 3.9389443128988946, + "tokens_seen": 472449024 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004327382146439318, + "loss": 2.9363, + "theoretical_loss": 3.938884890556486, + "tokens_seen": 472514560 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432728184553661, + "loss": 2.9786, + "theoretical_loss": 3.9388254787624617, + "tokens_seen": 472580096 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043271815446339017, + "loss": 2.8273, + "theoretical_loss": 3.938766077513486, + "tokens_seen": 472645632 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043270812437311935, + "loss": 2.8662, + "theoretical_loss": 3.938706686806226, + "tokens_seen": 472711168 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326980942828486, + "loss": 2.9246, + "theoretical_loss": 3.9386473066373497, + "tokens_seen": 472776704 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326880641925777, + "loss": 2.9862, + "theoretical_loss": 3.9385879370035264, + "tokens_seen": 472842240 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043267803410230695, + "loss": 2.6308, + "theoretical_loss": 3.938528577901427, + "tokens_seen": 472907776 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326680040120361, + "loss": 2.9238, + "theoretical_loss": 3.9384692293277253, + "tokens_seen": 472973312 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326579739217653, + "loss": 2.7366, + "theoretical_loss": 3.938409891279094, + "tokens_seen": 473038848 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326479438314945, + "loss": 2.6296, + "theoretical_loss": 3.93835056375221, + "tokens_seen": 473104384 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326379137412237, + "loss": 2.8136, + "theoretical_loss": 3.9382912467437503, + "tokens_seen": 473169920 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043262788365095286, + "loss": 2.6962, + "theoretical_loss": 3.938231940250393, + "tokens_seen": 473235456 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043261785356068204, + "loss": 2.8113, + "theoretical_loss": 3.9381726442688185, + "tokens_seen": 473300992 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004326078234704112, + "loss": 2.842, + "theoretical_loss": 3.938113358795709, + "tokens_seen": 473366528 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043259779338014045, + "loss": 2.8583, + "theoretical_loss": 3.9380540838277467, + "tokens_seen": 473432064 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 273042, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9048469066619873, + "objective/train/theoretical_loss": 3.9379948193616174, + "objective/train/tokens_used": 493957600, + "theoretical_loss": 3.9379948193616174, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325877632898696, + "loss": 2.6338, + "theoretical_loss": 3.9379948193616174, + "tokens_seen": 473497600 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325777331995988, + "loss": 2.8322, + "theoretical_loss": 3.9379355653940062, + "tokens_seen": 473563136 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432567703109328, + "loss": 2.8898, + "theoretical_loss": 3.9378763219216015, + "tokens_seen": 473628672 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325576730190572, + "loss": 2.6278, + "theoretical_loss": 3.937817088941093, + "tokens_seen": 473694208 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043254764292878636, + "loss": 2.7849, + "theoretical_loss": 3.9377578664491706, + "tokens_seen": 473759744 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043253761283851554, + "loss": 2.7501, + "theoretical_loss": 3.937698654442527, + "tokens_seen": 473825280 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325275827482447, + "loss": 2.8944, + "theoretical_loss": 3.937639452917856, + "tokens_seen": 473890816 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043251755265797396, + "loss": 3.0332, + "theoretical_loss": 3.9375802618718523, + "tokens_seen": 473956352 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004325075225677031, + "loss": 2.9932, + "theoretical_loss": 3.9375210813012127, + "tokens_seen": 474021888 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324974924774323, + "loss": 2.9509, + "theoretical_loss": 3.937461911202636, + "tokens_seen": 474087424 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043248746238716145, + "loss": 2.8843, + "theoretical_loss": 3.9374027515728214, + "tokens_seen": 474152960 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324774322968907, + "loss": 3.1177, + "theoretical_loss": 3.93734360240847, + "tokens_seen": 474218496 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043246740220661986, + "loss": 2.8071, + "theoretical_loss": 3.937284463706286, + "tokens_seen": 474284032 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043245737211634904, + "loss": 2.8188, + "theoretical_loss": 3.937225335462971, + "tokens_seen": 474349568 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324473420260782, + "loss": 2.8953, + "theoretical_loss": 3.937166217675233, + "tokens_seen": 474415104 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324373119358074, + "loss": 2.719, + "theoretical_loss": 3.9371071103397783, + "tokens_seen": 474480640 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043242728184553664, + "loss": 2.6903, + "theoretical_loss": 3.9370480134533157, + "tokens_seen": 474546176 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004324172517552658, + "loss": 2.7762, + "theoretical_loss": 3.9369889270125547, + "tokens_seen": 474611712 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432407221664995, + "loss": 2.5037, + "theoretical_loss": 3.9369298510142077, + "tokens_seen": 474677248 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323971915747242, + "loss": 3.0455, + "theoretical_loss": 3.936870785454988, + "tokens_seen": 474742784 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323871614844534, + "loss": 2.8792, + "theoretical_loss": 3.93681173033161, + "tokens_seen": 474808320 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043237713139418255, + "loss": 2.7244, + "theoretical_loss": 3.9367526856407897, + "tokens_seen": 474873856 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323671013039118, + "loss": 2.9051, + "theoretical_loss": 3.9366936513792448, + "tokens_seen": 474939392 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323570712136409, + "loss": 2.7048, + "theoretical_loss": 3.936634627543694, + "tokens_seen": 475004928 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043234704112337014, + "loss": 2.9354, + "theoretical_loss": 3.9365756141308594, + "tokens_seen": 475070464 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 273516, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.271859645843506, + "objective/train/theoretical_loss": 3.9365166111374608, + "objective/train/tokens_used": 495596000, + "theoretical_loss": 3.9365166111374608, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323370110330993, + "loss": 2.8993, + "theoretical_loss": 3.9365166111374608, + "tokens_seen": 475136000 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323269809428285, + "loss": 2.8789, + "theoretical_loss": 3.9364576185602234, + "tokens_seen": 475201536 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004323169508525577, + "loss": 2.9238, + "theoretical_loss": 3.9363986363958725, + "tokens_seen": 475267072 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043230692076228687, + "loss": 2.7607, + "theoretical_loss": 3.936339664641133, + "tokens_seen": 475332608 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043229689067201605, + "loss": 2.8522, + "theoretical_loss": 3.9362807032927343, + "tokens_seen": 475398144 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322868605817453, + "loss": 2.9343, + "theoretical_loss": 3.9362217523474046, + "tokens_seen": 475463680 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322768304914744, + "loss": 3.034, + "theoretical_loss": 3.9361628118018768, + "tokens_seen": 475529216 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043226680040120365, + "loss": 2.9514, + "theoretical_loss": 3.936103881652881, + "tokens_seen": 475594752 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043225677031093283, + "loss": 2.8471, + "theoretical_loss": 3.936044961897153, + "tokens_seen": 475660288 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432246740220662, + "loss": 2.8487, + "theoretical_loss": 3.9359860525314274, + "tokens_seen": 475725824 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322367101303912, + "loss": 2.7933, + "theoretical_loss": 3.9359271535524414, + "tokens_seen": 475791360 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043222668004012037, + "loss": 2.7604, + "theoretical_loss": 3.935868264956932, + "tokens_seen": 475856896 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043221664994984955, + "loss": 2.8025, + "theoretical_loss": 3.9358093867416413, + "tokens_seen": 475922432 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004322066198595788, + "loss": 2.9206, + "theoretical_loss": 3.935750518903309, + "tokens_seen": 475987968 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321965897693079, + "loss": 3.1348, + "theoretical_loss": 3.9356916614386783, + "tokens_seen": 476053504 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043218655967903715, + "loss": 2.8289, + "theoretical_loss": 3.9356328143444923, + "tokens_seen": 476119040 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321765295887663, + "loss": 2.77, + "theoretical_loss": 3.9355739776174987, + "tokens_seen": 476184576 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321664994984955, + "loss": 2.9714, + "theoretical_loss": 3.935515151254444, + "tokens_seen": 476250112 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321564694082247, + "loss": 3.0004, + "theoretical_loss": 3.935456335252076, + "tokens_seen": 476315648 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321464393179539, + "loss": 2.7851, + "theoretical_loss": 3.9353975296071453, + "tokens_seen": 476381184 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043213640922768306, + "loss": 2.9986, + "theoretical_loss": 3.9353387343164035, + "tokens_seen": 476446720 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043212637913741224, + "loss": 2.8154, + "theoretical_loss": 3.9352799493766035, + "tokens_seen": 476512256 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004321163490471414, + "loss": 2.9255, + "theoretical_loss": 3.9352211747845, + "tokens_seen": 476577792 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043210631895687065, + "loss": 2.792, + "theoretical_loss": 3.935162410536849, + "tokens_seen": 476643328 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320962888665998, + "loss": 2.5768, + "theoretical_loss": 3.9351036566304076, + "tokens_seen": 476708864 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 274597, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.601290225982666, + "objective/train/theoretical_loss": 3.935044913061936, + "objective/train/tokens_used": 497234400, + "theoretical_loss": 3.935044913061936, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.000432086258776329, + "loss": 2.8793, + "theoretical_loss": 3.935044913061936, + "tokens_seen": 476774400 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320762286860582, + "loss": 2.7909, + "theoretical_loss": 3.9349861798281918, + "tokens_seen": 476839936 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320661985957874, + "loss": 2.4871, + "theoretical_loss": 3.934927456925939, + "tokens_seen": 476905472 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043205616850551656, + "loss": 2.6598, + "theoretical_loss": 3.934868744351941, + "tokens_seen": 476971008 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043204613841524574, + "loss": 2.6697, + "theoretical_loss": 3.934810042102961, + "tokens_seen": 477036544 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320361083249749, + "loss": 2.8944, + "theoretical_loss": 3.934751350175766, + "tokens_seen": 477102080 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043202607823470416, + "loss": 2.8022, + "theoretical_loss": 3.9346926685671235, + "tokens_seen": 477167616 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320160481444333, + "loss": 2.9025, + "theoretical_loss": 3.934633997273803, + "tokens_seen": 477233152 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004320060180541625, + "loss": 2.8173, + "theoretical_loss": 3.934575336292575, + "tokens_seen": 477298688 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043199598796389165, + "loss": 2.9114, + "theoretical_loss": 3.9345166856202107, + "tokens_seen": 477364224 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319859578736209, + "loss": 2.9045, + "theoretical_loss": 3.9344580452534847, + "tokens_seen": 477429760 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043197592778335006, + "loss": 2.8471, + "theoretical_loss": 3.934399415189171, + "tokens_seen": 477495296 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043196589769307924, + "loss": 2.6342, + "theoretical_loss": 3.9343407954240464, + "tokens_seen": 477560832 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319558676028084, + "loss": 2.8732, + "theoretical_loss": 3.934282185954889, + "tokens_seen": 477626368 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319458375125376, + "loss": 2.5875, + "theoretical_loss": 3.934223586778477, + "tokens_seen": 477691904 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319358074222668, + "loss": 2.9511, + "theoretical_loss": 3.9341649978915925, + "tokens_seen": 477757440 + }, + { + "epoch": 0.14, + "learning_rate": 0.000431925777331996, + "loss": 2.8133, + "theoretical_loss": 3.9341064192910165, + "tokens_seen": 477822976 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043191574724172515, + "loss": 2.9208, + "theoretical_loss": 3.9340478509735326, + "tokens_seen": 477888512 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004319057171514544, + "loss": 2.8644, + "theoretical_loss": 3.9339892929359266, + "tokens_seen": 477954048 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043189568706118357, + "loss": 2.9012, + "theoretical_loss": 3.933930745174985, + "tokens_seen": 478019584 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043188565697091275, + "loss": 2.5838, + "theoretical_loss": 3.9338722076874957, + "tokens_seen": 478085120 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043187562688064193, + "loss": 2.7064, + "theoretical_loss": 3.933813680470247, + "tokens_seen": 478150656 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318655967903711, + "loss": 2.9544, + "theoretical_loss": 3.9337551635200314, + "tokens_seen": 478216192 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318555667001003, + "loss": 2.7427, + "theoretical_loss": 3.93369665683364, + "tokens_seen": 478281728 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318455366098295, + "loss": 2.8325, + "theoretical_loss": 3.9336381604078667, + "tokens_seen": 478347264 + }, + { + "epoch": 0.14, + "objective/train/docs_used": 275931, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.969980239868164, + "objective/train/theoretical_loss": 3.933579674239507, + "objective/train/tokens_used": 498872800, + "theoretical_loss": 3.933579674239507, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.00043183550651955865, + "loss": 2.9991, + "theoretical_loss": 3.933579674239507, + "tokens_seen": 478412800 + }, + { + "epoch": 0.14, + "learning_rate": 0.0004318254764292879, + "loss": 2.8388, + "theoretical_loss": 3.933521198325357, + "tokens_seen": 478478336 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431815446339017, + "loss": 2.872, + "theoretical_loss": 3.933462732662215, + "tokens_seen": 478543872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043180541624874625, + "loss": 2.9391, + "theoretical_loss": 3.933404277246881, + "tokens_seen": 478609408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043179538615847543, + "loss": 2.7832, + "theoretical_loss": 3.9333458320761547, + "tokens_seen": 478674944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317853560682046, + "loss": 2.8224, + "theoretical_loss": 3.93328739714684, + "tokens_seen": 478740480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317753259779338, + "loss": 2.6799, + "theoretical_loss": 3.9332289724557397, + "tokens_seen": 478806016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043176529588766303, + "loss": 2.6965, + "theoretical_loss": 3.9331705579996585, + "tokens_seen": 478871552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043175526579739215, + "loss": 2.9734, + "theoretical_loss": 3.9331121537754044, + "tokens_seen": 478937088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317452357071214, + "loss": 2.726, + "theoretical_loss": 3.933053759779784, + "tokens_seen": 479002624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317352056168505, + "loss": 2.5974, + "theoretical_loss": 3.9329953760096084, + "tokens_seen": 479068160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043172517552657975, + "loss": 2.7127, + "theoretical_loss": 3.9329370024616876, + "tokens_seen": 479133696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043171514543630893, + "loss": 2.7597, + "theoretical_loss": 3.9328786391328343, + "tokens_seen": 479199232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004317051153460381, + "loss": 2.9107, + "theoretical_loss": 3.9328202860198616, + "tokens_seen": 479264768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316950852557673, + "loss": 2.9456, + "theoretical_loss": 3.9327619431195853, + "tokens_seen": 479330304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316850551654965, + "loss": 2.769, + "theoretical_loss": 3.9327036104288227, + "tokens_seen": 479395840 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316750250752257, + "loss": 3.0487, + "theoretical_loss": 3.932645287944391, + "tokens_seen": 479461376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316649949849549, + "loss": 3.0331, + "theoretical_loss": 3.932586975663109, + "tokens_seen": 479526912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316549648946841, + "loss": 3.109, + "theoretical_loss": 3.9325286735817997, + "tokens_seen": 479592448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043164493480441326, + "loss": 2.7279, + "theoretical_loss": 3.9324703816972844, + "tokens_seen": 479657984 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043163490471414244, + "loss": 2.7961, + "theoretical_loss": 3.9324121000063865, + "tokens_seen": 479723520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004316248746238716, + "loss": 2.6693, + "theoretical_loss": 3.9323538285059314, + "tokens_seen": 479789056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043161484453360085, + "loss": 2.9624, + "theoretical_loss": 3.932295567192747, + "tokens_seen": 479854592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043160481444333, + "loss": 2.8733, + "theoretical_loss": 3.932237316063659, + "tokens_seen": 479920128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315947843530592, + "loss": 2.8734, + "theoretical_loss": 3.9321790751154984, + "tokens_seen": 479985664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 276348, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0607144832611084, + "objective/train/theoretical_loss": 3.9321208443450963, + "objective/train/tokens_used": 500511200, + "theoretical_loss": 3.9321208443450963, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315847542627884, + "loss": 2.938, + "theoretical_loss": 3.9321208443450963, + "tokens_seen": 480051200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315747241725176, + "loss": 2.5545, + "theoretical_loss": 3.9320626237492844, + "tokens_seen": 480116736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043156469408224676, + "loss": 2.6545, + "theoretical_loss": 3.9320044133248966, + "tokens_seen": 480182272 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043155466399197594, + "loss": 2.8061, + "theoretical_loss": 3.931946213068768, + "tokens_seen": 480247808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315446339017051, + "loss": 2.6674, + "theoretical_loss": 3.931888022977735, + "tokens_seen": 480313344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043153460381143436, + "loss": 2.9156, + "theoretical_loss": 3.9318298430486367, + "tokens_seen": 480378880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315245737211635, + "loss": 2.8293, + "theoretical_loss": 3.931771673278311, + "tokens_seen": 480444416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004315145436308927, + "loss": 3.0808, + "theoretical_loss": 3.9317135136635994, + "tokens_seen": 480509952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043150451354062185, + "loss": 2.7806, + "theoretical_loss": 3.931655364201344, + "tokens_seen": 480575488 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314944834503511, + "loss": 2.9279, + "theoretical_loss": 3.931597224888389, + "tokens_seen": 480641024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043148445336008026, + "loss": 3.0468, + "theoretical_loss": 3.931539095721578, + "tokens_seen": 480706560 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043147442326980944, + "loss": 3.0113, + "theoretical_loss": 3.9314809766977596, + "tokens_seen": 480772096 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314643931795386, + "loss": 2.9067, + "theoretical_loss": 3.93142286781378, + "tokens_seen": 480837632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314543630892678, + "loss": 2.8236, + "theoretical_loss": 3.931364769066489, + "tokens_seen": 480903168 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431444332998997, + "loss": 3.1, + "theoretical_loss": 3.931306680452737, + "tokens_seen": 480968704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314343029087262, + "loss": 3.1625, + "theoretical_loss": 3.9312486019693766, + "tokens_seen": 481034240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043142427281845535, + "loss": 2.8898, + "theoretical_loss": 3.931190533613261, + "tokens_seen": 481099776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004314142427281846, + "loss": 2.9089, + "theoretical_loss": 3.9311324753812453, + "tokens_seen": 481165312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043140421263791377, + "loss": 2.8168, + "theoretical_loss": 3.931074427270186, + "tokens_seen": 481230848 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043139418254764295, + "loss": 2.9268, + "theoretical_loss": 3.93101638927694, + "tokens_seen": 481296384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043138415245737213, + "loss": 2.8559, + "theoretical_loss": 3.9309583613983676, + "tokens_seen": 481361920 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313741223671013, + "loss": 2.8534, + "theoretical_loss": 3.930900343631328, + "tokens_seen": 481427456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313640922768305, + "loss": 2.7964, + "theoretical_loss": 3.9308423359726845, + "tokens_seen": 481492992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313540621865597, + "loss": 2.8632, + "theoretical_loss": 3.930784338419299, + "tokens_seen": 481558528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043134403209628885, + "loss": 2.6951, + "theoretical_loss": 3.9307263509680372, + "tokens_seen": 481624064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 277290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.784728765487671, + "objective/train/theoretical_loss": 3.930668373615765, + "objective/train/tokens_used": 502149600, + "theoretical_loss": 3.930668373615765, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313340020060181, + "loss": 2.8966, + "theoretical_loss": 3.930668373615765, + "tokens_seen": 481689600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004313239719157472, + "loss": 3.0139, + "theoretical_loss": 3.9306104063593503, + "tokens_seen": 481755136 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043131394182547645, + "loss": 2.8475, + "theoretical_loss": 3.9305524491956616, + "tokens_seen": 481820672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043130391173520563, + "loss": 2.8799, + "theoretical_loss": 3.9304945021215687, + "tokens_seen": 481886208 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312938816449348, + "loss": 2.8447, + "theoretical_loss": 3.9304365651339443, + "tokens_seen": 481951744 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431283851554664, + "loss": 3.0417, + "theoretical_loss": 3.9303786382296604, + "tokens_seen": 482017280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043127382146439323, + "loss": 2.7033, + "theoretical_loss": 3.930320721405593, + "tokens_seen": 482082816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043126379137412236, + "loss": 2.8143, + "theoretical_loss": 3.9302628146586165, + "tokens_seen": 482148352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312537612838516, + "loss": 3.1353, + "theoretical_loss": 3.930204917985609, + "tokens_seen": 482213888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312437311935807, + "loss": 2.8771, + "theoretical_loss": 3.9301470313834495, + "tokens_seen": 482279424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043123370110330995, + "loss": 3.1197, + "theoretical_loss": 3.9300891548490178, + "tokens_seen": 482344960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043122367101303913, + "loss": 2.7898, + "theoretical_loss": 3.9300312883791944, + "tokens_seen": 482410496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312136409227683, + "loss": 2.7825, + "theoretical_loss": 3.929973431970863, + "tokens_seen": 482476032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004312036108324975, + "loss": 3.0488, + "theoretical_loss": 3.9299155856209085, + "tokens_seen": 482541568 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311935807422267, + "loss": 2.6776, + "theoretical_loss": 3.9298577493262155, + "tokens_seen": 482607104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043118355065195586, + "loss": 2.9107, + "theoretical_loss": 3.9297999230836713, + "tokens_seen": 482672640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311735205616851, + "loss": 2.9791, + "theoretical_loss": 3.9297421068901643, + "tokens_seen": 482738176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311634904714142, + "loss": 2.7728, + "theoretical_loss": 3.9296843007425846, + "tokens_seen": 482803712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043115346038114346, + "loss": 2.9737, + "theoretical_loss": 3.9296265046378225, + "tokens_seen": 482869248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311434302908726, + "loss": 2.8517, + "theoretical_loss": 3.9295687185727717, + "tokens_seen": 482934784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311334002006018, + "loss": 2.7383, + "theoretical_loss": 3.929510942544325, + "tokens_seen": 483000320 + }, + { + "epoch": 0.15, + "learning_rate": 0.000431123370110331, + "loss": 3.0246, + "theoretical_loss": 3.9294531765493788, + "tokens_seen": 483065856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004311133400200602, + "loss": 2.8883, + "theoretical_loss": 3.9293954205848296, + "tokens_seen": 483131392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043110330992978936, + "loss": 2.4493, + "theoretical_loss": 3.9293376746475746, + "tokens_seen": 483196928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310932798395186, + "loss": 3.0786, + "theoretical_loss": 3.9292799387345143, + "tokens_seen": 483262464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 277804, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7938942909240723, + "objective/train/theoretical_loss": 3.929222212842549, + "objective/train/tokens_used": 503788000, + "theoretical_loss": 3.929222212842549, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310832497492477, + "loss": 3.0089, + "theoretical_loss": 3.929222212842549, + "tokens_seen": 483328000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043107321965897696, + "loss": 2.86, + "theoretical_loss": 3.929164496968581, + "tokens_seen": 483393536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310631895687061, + "loss": 2.8599, + "theoretical_loss": 3.929106791109514, + "tokens_seen": 483459072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310531594784353, + "loss": 3.1346, + "theoretical_loss": 3.9290490952622523, + "tokens_seen": 483524608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310431293881645, + "loss": 2.5788, + "theoretical_loss": 3.9289914094237037, + "tokens_seen": 483590144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310330992978937, + "loss": 3.0451, + "theoretical_loss": 3.9289337335907746, + "tokens_seen": 483655680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043102306920762286, + "loss": 2.8503, + "theoretical_loss": 3.928876067760375, + "tokens_seen": 483721216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043101303911735205, + "loss": 2.7226, + "theoretical_loss": 3.9288184119294147, + "tokens_seen": 483786752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004310030090270812, + "loss": 2.841, + "theoretical_loss": 3.928760766094806, + "tokens_seen": 483852288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043099297893681046, + "loss": 2.9864, + "theoretical_loss": 3.928703130253462, + "tokens_seen": 483917824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309829488465396, + "loss": 2.8682, + "theoretical_loss": 3.9286455044022968, + "tokens_seen": 483983360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309729187562688, + "loss": 2.7043, + "theoretical_loss": 3.9285878885382273, + "tokens_seen": 484048896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043096288866599795, + "loss": 2.4423, + "theoretical_loss": 3.9285302826581696, + "tokens_seen": 484114432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309528585757272, + "loss": 2.6732, + "theoretical_loss": 3.9284726867590436, + "tokens_seen": 484179968 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043094282848545637, + "loss": 2.9397, + "theoretical_loss": 3.9284151008377686, + "tokens_seen": 484245504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043093279839518555, + "loss": 2.6799, + "theoretical_loss": 3.9283575248912666, + "tokens_seen": 484311040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004309227683049148, + "loss": 2.4629, + "theoretical_loss": 3.92829995891646, + "tokens_seen": 484376576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043091273821464397, + "loss": 2.809, + "theoretical_loss": 3.928242402910273, + "tokens_seen": 484442112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043090270812437315, + "loss": 2.5356, + "theoretical_loss": 3.928184856869632, + "tokens_seen": 484507648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043089267803410233, + "loss": 2.8043, + "theoretical_loss": 3.9281273207914618, + "tokens_seen": 484573184 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308826479438315, + "loss": 2.8214, + "theoretical_loss": 3.928069794672693, + "tokens_seen": 484638720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308726178535607, + "loss": 2.8554, + "theoretical_loss": 3.928012278510254, + "tokens_seen": 484704256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308625877632899, + "loss": 2.7812, + "theoretical_loss": 3.9279547723010753, + "tokens_seen": 484769792 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043085255767301905, + "loss": 3.1759, + "theoretical_loss": 3.9278972760420903, + "tokens_seen": 484835328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308425275827483, + "loss": 2.7076, + "theoretical_loss": 3.927839789730232, + "tokens_seen": 484900864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 279062, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1616718769073486, + "objective/train/theoretical_loss": 3.9277823133624366, + "objective/train/tokens_used": 505426400, + "theoretical_loss": 3.9277823133624366, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004308324974924774, + "loss": 2.8253, + "theoretical_loss": 3.9277823133624366, + "tokens_seen": 484966400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043082246740220665, + "loss": 2.749, + "theoretical_loss": 3.927724846935639, + "tokens_seen": 485031936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043081243731193583, + "loss": 2.7862, + "theoretical_loss": 3.9276673904467776, + "tokens_seen": 485097472 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430802407221665, + "loss": 2.8595, + "theoretical_loss": 3.927609943892792, + "tokens_seen": 485163008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307923771313942, + "loss": 2.7832, + "theoretical_loss": 3.9275525072706223, + "tokens_seen": 485228544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043078234704112343, + "loss": 2.7835, + "theoretical_loss": 3.9274950805772098, + "tokens_seen": 485294080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043077231695085256, + "loss": 2.7645, + "theoretical_loss": 3.9274376638094983, + "tokens_seen": 485359616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307622868605818, + "loss": 2.8211, + "theoretical_loss": 3.9273802569644323, + "tokens_seen": 485425152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307522567703109, + "loss": 2.6924, + "theoretical_loss": 3.9273228600389585, + "tokens_seen": 485490688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043074222668004015, + "loss": 2.7889, + "theoretical_loss": 3.9272654730300225, + "tokens_seen": 485556224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043073219658976933, + "loss": 3.0397, + "theoretical_loss": 3.9272080959345734, + "tokens_seen": 485621760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307221664994985, + "loss": 2.699, + "theoretical_loss": 3.927150728749562, + "tokens_seen": 485687296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307121364092277, + "loss": 2.729, + "theoretical_loss": 3.9270933714719396, + "tokens_seen": 485752832 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004307021063189569, + "loss": 2.7437, + "theoretical_loss": 3.9270360240986575, + "tokens_seen": 485818368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043069207622868606, + "loss": 2.8264, + "theoretical_loss": 3.926978686626671, + "tokens_seen": 485883904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306820461384153, + "loss": 2.9605, + "theoretical_loss": 3.926921359052935, + "tokens_seen": 485949440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306720160481444, + "loss": 3.0128, + "theoretical_loss": 3.9268640413744063, + "tokens_seen": 486014976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043066198595787366, + "loss": 2.7019, + "theoretical_loss": 3.9268067335880428, + "tokens_seen": 486080512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306519558676028, + "loss": 2.9512, + "theoretical_loss": 3.9267494356908044, + "tokens_seen": 486146048 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430641925777332, + "loss": 2.9346, + "theoretical_loss": 3.926692147679651, + "tokens_seen": 486211584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306318956870612, + "loss": 2.6555, + "theoretical_loss": 3.926634869551545, + "tokens_seen": 486277120 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306218655967904, + "loss": 2.9892, + "theoretical_loss": 3.9265776013034497, + "tokens_seen": 486342656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043061183550651956, + "loss": 2.7261, + "theoretical_loss": 3.9265203429323305, + "tokens_seen": 486408192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004306018054162488, + "loss": 2.6338, + "theoretical_loss": 3.926463094435153, + "tokens_seen": 486473728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305917753259779, + "loss": 2.7808, + "theoretical_loss": 3.9264058558088846, + "tokens_seen": 486539264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 279686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.470508337020874, + "objective/train/theoretical_loss": 3.926348627050494, + "objective/train/tokens_used": 507064800, + "theoretical_loss": 3.926348627050494, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043058174523570716, + "loss": 2.8973, + "theoretical_loss": 3.926348627050494, + "tokens_seen": 486604800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305717151454363, + "loss": 2.9057, + "theoretical_loss": 3.9262914081569518, + "tokens_seen": 486670336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305616850551655, + "loss": 3.0043, + "theoretical_loss": 3.9262341991252296, + "tokens_seen": 486735872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305516549648947, + "loss": 2.6892, + "theoretical_loss": 3.9261769999522995, + "tokens_seen": 486801408 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305416248746239, + "loss": 2.7105, + "theoretical_loss": 3.926119810635136, + "tokens_seen": 486866944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043053159478435306, + "loss": 3.0511, + "theoretical_loss": 3.9260626311707143, + "tokens_seen": 486932480 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043052156469408225, + "loss": 2.6944, + "theoretical_loss": 3.926005461556011, + "tokens_seen": 486998016 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004305115346038114, + "loss": 2.5711, + "theoretical_loss": 3.9259483017880052, + "tokens_seen": 487063552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043050150451354066, + "loss": 2.9071, + "theoretical_loss": 3.9258911518636763, + "tokens_seen": 487129088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304914744232698, + "loss": 2.9919, + "theoretical_loss": 3.9258340117800037, + "tokens_seen": 487194624 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430481444332999, + "loss": 2.6231, + "theoretical_loss": 3.925776881533971, + "tokens_seen": 487260160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043047141424272815, + "loss": 3.1136, + "theoretical_loss": 3.9257197611225614, + "tokens_seen": 487325696 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304613841524574, + "loss": 2.8313, + "theoretical_loss": 3.925662650542759, + "tokens_seen": 487391232 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043045135406218657, + "loss": 2.8077, + "theoretical_loss": 3.92560554979155, + "tokens_seen": 487456768 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043044132397191575, + "loss": 2.933, + "theoretical_loss": 3.925548458865923, + "tokens_seen": 487522304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043043129388164493, + "loss": 2.7352, + "theoretical_loss": 3.925491377762866, + "tokens_seen": 487587840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043042126379137417, + "loss": 2.9462, + "theoretical_loss": 3.925434306479369, + "tokens_seen": 487653376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004304112337011033, + "loss": 2.8203, + "theoretical_loss": 3.925377245012424, + "tokens_seen": 487718912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043040120361083253, + "loss": 2.6397, + "theoretical_loss": 3.9253201933590227, + "tokens_seen": 487784448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043039117352056165, + "loss": 2.8775, + "theoretical_loss": 3.92526315151616, + "tokens_seen": 487849984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303811434302909, + "loss": 2.7412, + "theoretical_loss": 3.9252061194808316, + "tokens_seen": 487915520 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043037111334002007, + "loss": 2.9406, + "theoretical_loss": 3.925149097250033, + "tokens_seen": 487981056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043036108324974925, + "loss": 2.9284, + "theoretical_loss": 3.925092084820764, + "tokens_seen": 488046592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043035105315947843, + "loss": 2.7134, + "theoretical_loss": 3.9250350821900226, + "tokens_seen": 488112128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303410230692076, + "loss": 2.7296, + "theoretical_loss": 3.92497808935481, + "tokens_seen": 488177664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 280726, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.50590443611145, + "objective/train/theoretical_loss": 3.9249211063121283, + "objective/train/tokens_used": 508703200, + "theoretical_loss": 3.9249211063121283, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303309929789368, + "loss": 3.0183, + "theoretical_loss": 3.9249211063121283, + "tokens_seen": 488243200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043032096288866603, + "loss": 2.6671, + "theoretical_loss": 3.9248641330589806, + "tokens_seen": 488308736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043031093279839516, + "loss": 2.8775, + "theoretical_loss": 3.924807169592372, + "tokens_seen": 488374272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004303009027081244, + "loss": 2.9192, + "theoretical_loss": 3.9247502159093086, + "tokens_seen": 488439808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302908726178535, + "loss": 2.6838, + "theoretical_loss": 3.924693272006796, + "tokens_seen": 488505344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043028084252758276, + "loss": 2.9602, + "theoretical_loss": 3.924636337881845, + "tokens_seen": 488570880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043027081243731194, + "loss": 2.9434, + "theoretical_loss": 3.924579413531465, + "tokens_seen": 488636416 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302607823470411, + "loss": 2.9598, + "theoretical_loss": 3.9245224989526664, + "tokens_seen": 488701952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302507522567703, + "loss": 2.862, + "theoretical_loss": 3.924465594142463, + "tokens_seen": 488767488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043024072216649953, + "loss": 2.9043, + "theoretical_loss": 3.9244086990978673, + "tokens_seen": 488833024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043023069207622866, + "loss": 2.8912, + "theoretical_loss": 3.924351813815895, + "tokens_seen": 488898560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004302206619859579, + "loss": 2.9308, + "theoretical_loss": 3.9242949382935635, + "tokens_seen": 488964096 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430210631895687, + "loss": 2.9168, + "theoretical_loss": 3.9242380725278894, + "tokens_seen": 489029632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043020060180541626, + "loss": 2.723, + "theoretical_loss": 3.9241812165158922, + "tokens_seen": 489095168 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043019057171514544, + "loss": 2.9341, + "theoretical_loss": 3.9241243702545923, + "tokens_seen": 489160704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301805416248746, + "loss": 2.7376, + "theoretical_loss": 3.9240675337410122, + "tokens_seen": 489226240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043017051153460386, + "loss": 2.9562, + "theoretical_loss": 3.924010706972174, + "tokens_seen": 489291776 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430160481444333, + "loss": 2.5616, + "theoretical_loss": 3.9239538899451025, + "tokens_seen": 489357312 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301504513540622, + "loss": 2.9733, + "theoretical_loss": 3.923897082656823, + "tokens_seen": 489422848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301404212637914, + "loss": 2.7338, + "theoretical_loss": 3.9238402851043634, + "tokens_seen": 489488384 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301303911735206, + "loss": 2.8406, + "theoretical_loss": 3.9237834972847505, + "tokens_seen": 489553920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043012036108324976, + "loss": 2.8607, + "theoretical_loss": 3.923726719195016, + "tokens_seen": 489619456 + }, + { + "epoch": 0.15, + "learning_rate": 0.000430110330992979, + "loss": 2.7782, + "theoretical_loss": 3.9236699508321884, + "tokens_seen": 489684992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004301003009027081, + "loss": 2.9235, + "theoretical_loss": 3.9236131921933017, + "tokens_seen": 489750528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043009027081243736, + "loss": 2.9848, + "theoretical_loss": 3.923556443275388, + "tokens_seen": 489816064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 281543, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0093557834625244, + "objective/train/theoretical_loss": 3.923499704075483, + "objective/train/tokens_used": 510341600, + "theoretical_loss": 3.923499704075483, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300802407221665, + "loss": 2.8803, + "theoretical_loss": 3.923499704075483, + "tokens_seen": 489881600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300702106318957, + "loss": 2.945, + "theoretical_loss": 3.923442974590623, + "tokens_seen": 489947136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300601805416249, + "loss": 2.7447, + "theoretical_loss": 3.923386254817845, + "tokens_seen": 490012672 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004300501504513541, + "loss": 3.1141, + "theoretical_loss": 3.9233295447541874, + "tokens_seen": 490078208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043004012036108326, + "loss": 2.761, + "theoretical_loss": 3.92327284439669, + "tokens_seen": 490143744 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043003009027081245, + "loss": 2.8582, + "theoretical_loss": 3.9232161537423957, + "tokens_seen": 490209280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043002006018054163, + "loss": 2.824, + "theoretical_loss": 3.9231594727883454, + "tokens_seen": 490274816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043001003009027086, + "loss": 2.6088, + "theoretical_loss": 3.9231028015315834, + "tokens_seen": 490340352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00043, + "loss": 2.857, + "theoretical_loss": 3.9230461399691556, + "tokens_seen": 490405888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299899699097292, + "loss": 2.7562, + "theoretical_loss": 3.922989488098107, + "tokens_seen": 490471424 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042997993981945835, + "loss": 2.9854, + "theoretical_loss": 3.9229328459154873, + "tokens_seen": 490536960 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299699097291876, + "loss": 2.8507, + "theoretical_loss": 3.9228762134183444, + "tokens_seen": 490602496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042995987963891677, + "loss": 3.0315, + "theoretical_loss": 3.922819590603729, + "tokens_seen": 490668032 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042994984954864595, + "loss": 2.8923, + "theoretical_loss": 3.922762977468692, + "tokens_seen": 490733568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042993981945837513, + "loss": 2.7771, + "theoretical_loss": 3.9227063740102874, + "tokens_seen": 490799104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042992978936810437, + "loss": 3.0867, + "theoretical_loss": 3.9226497802255684, + "tokens_seen": 490864640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004299197592778335, + "loss": 2.6551, + "theoretical_loss": 3.922593196111592, + "tokens_seen": 490930176 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042990972918756273, + "loss": 2.952, + "theoretical_loss": 3.9225366216654134, + "tokens_seen": 490995712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042989969909729185, + "loss": 2.7502, + "theoretical_loss": 3.9224800568840914, + "tokens_seen": 491061248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298896690070211, + "loss": 2.7318, + "theoretical_loss": 3.922423501764686, + "tokens_seen": 491126784 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042987963891675027, + "loss": 3.0758, + "theoretical_loss": 3.9223669563042565, + "tokens_seen": 491192320 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042986960882647945, + "loss": 2.9801, + "theoretical_loss": 3.9223104204998664, + "tokens_seen": 491257856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042985957873620863, + "loss": 2.9669, + "theoretical_loss": 3.922253894348578, + "tokens_seen": 491323392 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298495486459378, + "loss": 2.9609, + "theoretical_loss": 3.922197377847456, + "tokens_seen": 491388928 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429839518555667, + "loss": 2.9549, + "theoretical_loss": 3.922140870993566, + "tokens_seen": 491454464 + }, + { + "debugging/Self-BLEU-5": 0.364302224771738, + "debugging/distinct-1-grams": 0.7813905547226386, + "debugging/distinct-2-grams": 0.9746492802208804, + "debugging/entropy-1-grams": 5.062092644878832, + "debugging/entropy-2-grams": 5.611156931113771, + "debugging/length": 539.0, + "debugging/num_segments": 4, + "epoch": 0.15, + "objective/train/docs_used": 282054, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8211379051208496, + "objective/train/theoretical_loss": 3.9220843737839752, + "objective/train/tokens_used": 511980000, + "theoretical_loss": 3.9220843737839752, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042982948846539623, + "loss": 2.85, + "theoretical_loss": 3.9220843737839752, + "tokens_seen": 491520000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042981945837512536, + "loss": 2.6656, + "theoretical_loss": 3.9220278862157523, + "tokens_seen": 491585536 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004298094282848546, + "loss": 2.8264, + "theoretical_loss": 3.921971408285967, + "tokens_seen": 491651072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297993981945837, + "loss": 2.8791, + "theoretical_loss": 3.92191493999169, + "tokens_seen": 491716608 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042978936810431296, + "loss": 2.8162, + "theoretical_loss": 3.921858481329994, + "tokens_seen": 491782144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042977933801404214, + "loss": 3.1575, + "theoretical_loss": 3.9218020322979514, + "tokens_seen": 491847680 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297693079237713, + "loss": 2.671, + "theoretical_loss": 3.9217455928926377, + "tokens_seen": 491913216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297592778335005, + "loss": 2.9984, + "theoretical_loss": 3.9216891631111293, + "tokens_seen": 491978752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042974924774322973, + "loss": 2.8436, + "theoretical_loss": 3.9216327429505027, + "tokens_seen": 492044288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042973921765295886, + "loss": 2.7374, + "theoretical_loss": 3.921576332407837, + "tokens_seen": 492109824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297291875626881, + "loss": 3.0636, + "theoretical_loss": 3.921519931480213, + "tokens_seen": 492175360 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004297191574724172, + "loss": 3.0291, + "theoretical_loss": 3.92146354016471, + "tokens_seen": 492240896 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042970912738214646, + "loss": 2.9755, + "theoretical_loss": 3.921407158458411, + "tokens_seen": 492306432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042969909729187564, + "loss": 2.7441, + "theoretical_loss": 3.9213507863584014, + "tokens_seen": 492371968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296890672016048, + "loss": 2.8323, + "theoretical_loss": 3.921294423861764, + "tokens_seen": 492437504 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429679037111334, + "loss": 2.925, + "theoretical_loss": 3.921238070965586, + "tokens_seen": 492503040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296690070210632, + "loss": 2.81, + "theoretical_loss": 3.921181727666955, + "tokens_seen": 492568576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042965897693079236, + "loss": 2.8812, + "theoretical_loss": 3.92112539396296, + "tokens_seen": 492634112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296489468405216, + "loss": 3.0883, + "theoretical_loss": 3.9210690698506907, + "tokens_seen": 492699648 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296389167502507, + "loss": 3.0465, + "theoretical_loss": 3.9210127553272383, + "tokens_seen": 492765184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042962888665997996, + "loss": 2.9085, + "theoretical_loss": 3.9209564503896956, + "tokens_seen": 492830720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296188565697091, + "loss": 2.6728, + "theoretical_loss": 3.9209001550351563, + "tokens_seen": 492896256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004296088264794383, + "loss": 2.7792, + "theoretical_loss": 3.9208438692607164, + "tokens_seen": 492961792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295987963891675, + "loss": 2.9342, + "theoretical_loss": 3.920787593063471, + "tokens_seen": 493027328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295887662988967, + "loss": 3.0404, + "theoretical_loss": 3.9207313264405186, + "tokens_seen": 493092864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 282706, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2857396602630615, + "objective/train/theoretical_loss": 3.9206750693889583, + "objective/train/tokens_used": 513618400, + "theoretical_loss": 3.9206750693889583, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042957873620862587, + "loss": 2.9551, + "theoretical_loss": 3.9206750693889583, + "tokens_seen": 493158400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295687061183551, + "loss": 2.7466, + "theoretical_loss": 3.9206188219058897, + "tokens_seen": 493223936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042955867602808423, + "loss": 2.7163, + "theoretical_loss": 3.9205625839884144, + "tokens_seen": 493289472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042954864593781347, + "loss": 2.7523, + "theoretical_loss": 3.920506355633636, + "tokens_seen": 493355008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295386158475426, + "loss": 2.9156, + "theoretical_loss": 3.9204501368386575, + "tokens_seen": 493420544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042952858575727183, + "loss": 2.8954, + "theoretical_loss": 3.9203939276005846, + "tokens_seen": 493486080 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429518555667001, + "loss": 2.885, + "theoretical_loss": 3.9203377279165235, + "tokens_seen": 493551616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004295085255767302, + "loss": 2.812, + "theoretical_loss": 3.920281537783582, + "tokens_seen": 493617152 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042949849548645937, + "loss": 2.797, + "theoretical_loss": 3.9202253571988694, + "tokens_seen": 493682688 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042948846539618855, + "loss": 2.858, + "theoretical_loss": 3.9201691861594963, + "tokens_seen": 493748224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042947843530591773, + "loss": 2.8976, + "theoretical_loss": 3.9201130246625735, + "tokens_seen": 493813760 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042946840521564697, + "loss": 2.9483, + "theoretical_loss": 3.920056872705214, + "tokens_seen": 493879296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294583751253761, + "loss": 2.8325, + "theoretical_loss": 3.9200007302845323, + "tokens_seen": 493944832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042944834503510533, + "loss": 2.9839, + "theoretical_loss": 3.9199445973976434, + "tokens_seen": 494010368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294383149448345, + "loss": 2.9204, + "theoretical_loss": 3.9198884740416644, + "tokens_seen": 494075904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004294282848545637, + "loss": 2.8143, + "theoretical_loss": 3.9198323602137126, + "tokens_seen": 494141440 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042941825476429293, + "loss": 3.0728, + "theoretical_loss": 3.9197762559109073, + "tokens_seen": 494206976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042940822467402206, + "loss": 2.9142, + "theoretical_loss": 3.919720161130368, + "tokens_seen": 494272512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293981945837513, + "loss": 3.0248, + "theoretical_loss": 3.9196640758692176, + "tokens_seen": 494338048 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042938816449348047, + "loss": 2.6867, + "theoretical_loss": 3.9196080001245783, + "tokens_seen": 494403584 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042937813440320965, + "loss": 2.8787, + "theoretical_loss": 3.9195519338935743, + "tokens_seen": 494469120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042936810431293883, + "loss": 2.6973, + "theoretical_loss": 3.919495877173331, + "tokens_seen": 494534656 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429358074222668, + "loss": 2.8302, + "theoretical_loss": 3.9194398299609743, + "tokens_seen": 494600192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293480441323972, + "loss": 3.1014, + "theoretical_loss": 3.919383792253633, + "tokens_seen": 494665728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042933801404212643, + "loss": 2.8187, + "theoretical_loss": 3.919327764048437, + "tokens_seen": 494731264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 283457, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3519198894500732, + "objective/train/theoretical_loss": 3.919271745342514, + "objective/train/tokens_used": 515256800, + "theoretical_loss": 3.919271745342514, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042932798395185556, + "loss": 2.9657, + "theoretical_loss": 3.919271745342514, + "tokens_seen": 494796800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293179538615848, + "loss": 2.8827, + "theoretical_loss": 3.9192157361329976, + "tokens_seen": 494862336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004293079237713139, + "loss": 2.8081, + "theoretical_loss": 3.9191597364170208, + "tokens_seen": 494927872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042929789368104316, + "loss": 2.9318, + "theoretical_loss": 3.919103746191716, + "tokens_seen": 494993408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042928786359077234, + "loss": 2.7932, + "theoretical_loss": 3.91904776545422, + "tokens_seen": 495058944 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292778335005015, + "loss": 2.8108, + "theoretical_loss": 3.918991794201669, + "tokens_seen": 495124480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292678034102307, + "loss": 2.684, + "theoretical_loss": 3.9189358324312007, + "tokens_seen": 495190016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042925777331995993, + "loss": 2.6035, + "theoretical_loss": 3.9188798801399543, + "tokens_seen": 495255552 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042924774322968906, + "loss": 2.8571, + "theoretical_loss": 3.9188239373250697, + "tokens_seen": 495321088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292377131394183, + "loss": 2.8791, + "theoretical_loss": 3.9187680039836885, + "tokens_seen": 495386624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004292276830491474, + "loss": 2.9489, + "theoretical_loss": 3.918712080112954, + "tokens_seen": 495452160 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042921765295887666, + "loss": 2.83, + "theoretical_loss": 3.9186561657100096, + "tokens_seen": 495517696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042920762286860584, + "loss": 2.736, + "theoretical_loss": 3.918600260772001, + "tokens_seen": 495583232 + }, + { + "epoch": 0.15, + "learning_rate": 0.000429197592778335, + "loss": 2.7423, + "theoretical_loss": 3.9185443652960745, + "tokens_seen": 495648768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291875626880642, + "loss": 3.0763, + "theoretical_loss": 3.918488479279378, + "tokens_seen": 495714304 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291775325977934, + "loss": 2.8709, + "theoretical_loss": 3.9184326027190606, + "tokens_seen": 495779840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042916750250752256, + "loss": 2.9463, + "theoretical_loss": 3.9183767356122714, + "tokens_seen": 495845376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291574724172518, + "loss": 3.1239, + "theoretical_loss": 3.9183208779561634, + "tokens_seen": 495910912 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291474423269809, + "loss": 2.9989, + "theoretical_loss": 3.9182650297478885, + "tokens_seen": 495976448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042913741223671016, + "loss": 2.8044, + "theoretical_loss": 3.9182091909846, + "tokens_seen": 496041984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291273821464393, + "loss": 2.9178, + "theoretical_loss": 3.9181533616634545, + "tokens_seen": 496107520 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291173520561685, + "loss": 3.0983, + "theoretical_loss": 3.918097541781607, + "tokens_seen": 496173056 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004291073219658977, + "loss": 3.0364, + "theoretical_loss": 3.9180417313362157, + "tokens_seen": 496238592 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290972918756269, + "loss": 2.9606, + "theoretical_loss": 3.9179859303244395, + "tokens_seen": 496304128 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042908726178535607, + "loss": 2.6999, + "theoretical_loss": 3.917930138743438, + "tokens_seen": 496369664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 284572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4066948890686035, + "objective/train/theoretical_loss": 3.9178743565903735, + "objective/train/tokens_used": 516895200, + "theoretical_loss": 3.9178743565903735, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290772316950853, + "loss": 2.7991, + "theoretical_loss": 3.9178743565903735, + "tokens_seen": 496435200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042906720160481443, + "loss": 2.9311, + "theoretical_loss": 3.9178185838624078, + "tokens_seen": 496500736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042905717151454367, + "loss": 2.6928, + "theoretical_loss": 3.917762820556705, + "tokens_seen": 496566272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290471414242728, + "loss": 2.8094, + "theoretical_loss": 3.9177070666704292, + "tokens_seen": 496631808 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042903711133400203, + "loss": 2.7391, + "theoretical_loss": 3.917651322200747, + "tokens_seen": 496697344 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290270812437312, + "loss": 2.8844, + "theoretical_loss": 3.9175955871448265, + "tokens_seen": 496762880 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004290170511534604, + "loss": 2.9062, + "theoretical_loss": 3.917539861499836, + "tokens_seen": 496828416 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042900702106318957, + "loss": 2.9001, + "theoretical_loss": 3.9174841452629456, + "tokens_seen": 496893952 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042899699097291875, + "loss": 3.0454, + "theoretical_loss": 3.9174284384313256, + "tokens_seen": 496959488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042898696088264793, + "loss": 2.9205, + "theoretical_loss": 3.917372741002149, + "tokens_seen": 497025024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042897693079237717, + "loss": 2.9715, + "theoretical_loss": 3.917317052972589, + "tokens_seen": 497090560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289669007021063, + "loss": 2.823, + "theoretical_loss": 3.9172613743398204, + "tokens_seen": 497156096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042895687061183553, + "loss": 2.7293, + "theoretical_loss": 3.91720570510102, + "tokens_seen": 497221632 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289468405215647, + "loss": 2.8304, + "theoretical_loss": 3.917150045253364, + "tokens_seen": 497287168 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289368104312939, + "loss": 2.9458, + "theoretical_loss": 3.917094394794031, + "tokens_seen": 497352704 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004289267803410231, + "loss": 3.1032, + "theoretical_loss": 3.917038753720201, + "tokens_seen": 497418240 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042891675025075226, + "loss": 2.8554, + "theoretical_loss": 3.9169831220290554, + "tokens_seen": 497483776 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042890672016048144, + "loss": 2.9893, + "theoretical_loss": 3.916927499717775, + "tokens_seen": 497549312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042889669007021067, + "loss": 2.8381, + "theoretical_loss": 3.916871886783544, + "tokens_seen": 497614848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288866599799398, + "loss": 2.9519, + "theoretical_loss": 3.9168162832235467, + "tokens_seen": 497680384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042887662988966903, + "loss": 3.1386, + "theoretical_loss": 3.9167606890349687, + "tokens_seen": 497745920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042886659979939816, + "loss": 2.5596, + "theoretical_loss": 3.9167051042149974, + "tokens_seen": 497811456 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288565697091274, + "loss": 2.9973, + "theoretical_loss": 3.9166495287608205, + "tokens_seen": 497876992 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288465396188566, + "loss": 2.8093, + "theoretical_loss": 3.9165939626696273, + "tokens_seen": 497942528 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042883650952858576, + "loss": 2.8571, + "theoretical_loss": 3.916538405938609, + "tokens_seen": 498008064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 285190, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.950747489929199, + "objective/train/theoretical_loss": 3.916482858564957, + "objective/train/tokens_used": 518533600, + "theoretical_loss": 3.916482858564957, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042882647943831494, + "loss": 2.9687, + "theoretical_loss": 3.916482858564957, + "tokens_seen": 498073600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288164493480441, + "loss": 2.7534, + "theoretical_loss": 3.916427320545864, + "tokens_seen": 498139136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004288064192577733, + "loss": 2.8904, + "theoretical_loss": 3.9163717918785252, + "tokens_seen": 498204672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042879638916750254, + "loss": 2.9187, + "theoretical_loss": 3.9163162725601355, + "tokens_seen": 498270208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042878635907723166, + "loss": 2.6975, + "theoretical_loss": 3.916260762587892, + "tokens_seen": 498335744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287763289869609, + "loss": 2.695, + "theoretical_loss": 3.916205261958991, + "tokens_seen": 498401280 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287662988966901, + "loss": 2.7584, + "theoretical_loss": 3.916149770670633, + "tokens_seen": 498466816 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042875626880641926, + "loss": 2.8199, + "theoretical_loss": 3.9160942887200183, + "tokens_seen": 498532352 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042874623871614844, + "loss": 2.6627, + "theoretical_loss": 3.916038816104348, + "tokens_seen": 498597888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287362086258776, + "loss": 2.9793, + "theoretical_loss": 3.9159833528208248, + "tokens_seen": 498663424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004287261785356068, + "loss": 2.6124, + "theoretical_loss": 3.915927898866653, + "tokens_seen": 498728960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042871614844533604, + "loss": 2.8027, + "theoretical_loss": 3.915872454239037, + "tokens_seen": 498794496 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042870611835506517, + "loss": 3.0653, + "theoretical_loss": 3.9158170189351833, + "tokens_seen": 498860032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286960882647944, + "loss": 3.1196, + "theoretical_loss": 3.9157615929523004, + "tokens_seen": 498925568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042868605817452353, + "loss": 2.8093, + "theoretical_loss": 3.9157061762875953, + "tokens_seen": 498991104 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042867602808425276, + "loss": 2.8654, + "theoretical_loss": 3.9156507689382796, + "tokens_seen": 499056640 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428665997993982, + "loss": 2.7376, + "theoretical_loss": 3.915595370901563, + "tokens_seen": 499122176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286559679037111, + "loss": 2.7512, + "theoretical_loss": 3.915539982174659, + "tokens_seen": 499187712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042864593781344036, + "loss": 3.015, + "theoretical_loss": 3.9154846027547805, + "tokens_seen": 499253248 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286359077231695, + "loss": 2.6258, + "theoretical_loss": 3.915429232639142, + "tokens_seen": 499318784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286258776328987, + "loss": 2.9637, + "theoretical_loss": 3.9153738718249604, + "tokens_seen": 499384320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286158475426279, + "loss": 2.8947, + "theoretical_loss": 3.9153185203094516, + "tokens_seen": 499449856 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004286058174523571, + "loss": 3.0437, + "theoretical_loss": 3.915263178089835, + "tokens_seen": 499515392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042859578736208627, + "loss": 3.0086, + "theoretical_loss": 3.915207845163329, + "tokens_seen": 499580928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285857572718155, + "loss": 2.7415, + "theoretical_loss": 3.915152521527155, + "tokens_seen": 499646464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 286367, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.678431272506714, + "objective/train/theoretical_loss": 3.915097207178535, + "objective/train/tokens_used": 520172000, + "theoretical_loss": 3.915097207178535, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042857572718154463, + "loss": 2.8241, + "theoretical_loss": 3.915097207178535, + "tokens_seen": 499712000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042856569709127387, + "loss": 2.8393, + "theoretical_loss": 3.9150419021146927, + "tokens_seen": 499777536 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428555667001003, + "loss": 2.9709, + "theoretical_loss": 3.9149866063328504, + "tokens_seen": 499843072 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042854563691073223, + "loss": 2.9225, + "theoretical_loss": 3.914931319830236, + "tokens_seen": 499908608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285356068204614, + "loss": 2.8198, + "theoretical_loss": 3.9148760426040736, + "tokens_seen": 499974144 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004285255767301906, + "loss": 2.7686, + "theoretical_loss": 3.9148207746515933, + "tokens_seen": 500039680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042851554663991977, + "loss": 2.7698, + "theoretical_loss": 3.9147655159700236, + "tokens_seen": 500105216 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042850551654964895, + "loss": 2.8607, + "theoretical_loss": 3.914710266556594, + "tokens_seen": 500170752 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042849548645937813, + "loss": 2.9713, + "theoretical_loss": 3.9146550264085374, + "tokens_seen": 500236288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042848545636910737, + "loss": 2.6836, + "theoretical_loss": 3.9145997955230842, + "tokens_seen": 500301824 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284754262788365, + "loss": 2.6966, + "theoretical_loss": 3.9145445738974702, + "tokens_seen": 500367360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042846539618856573, + "loss": 2.6121, + "theoretical_loss": 3.91448936152893, + "tokens_seen": 500432896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284553660982949, + "loss": 2.7794, + "theoretical_loss": 3.914434158414699, + "tokens_seen": 500498432 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284453360080241, + "loss": 3.1444, + "theoretical_loss": 3.9143789645520153, + "tokens_seen": 500563968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004284353059177533, + "loss": 2.7484, + "theoretical_loss": 3.914323779938117, + "tokens_seen": 500629504 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042842527582748246, + "loss": 2.6557, + "theoretical_loss": 3.9142686045702444, + "tokens_seen": 500695040 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042841524573721164, + "loss": 2.847, + "theoretical_loss": 3.914213438445638, + "tokens_seen": 500760576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042840521564694087, + "loss": 3.133, + "theoretical_loss": 3.9141582815615408, + "tokens_seen": 500826112 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042839518555667, + "loss": 2.9062, + "theoretical_loss": 3.9141031339151953, + "tokens_seen": 500891648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042838515546639923, + "loss": 2.8711, + "theoretical_loss": 3.9140479955038456, + "tokens_seen": 500957184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042837512537612836, + "loss": 2.7578, + "theoretical_loss": 3.9139928663247385, + "tokens_seen": 501022720 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283650952858576, + "loss": 2.7539, + "theoretical_loss": 3.9139377463751197, + "tokens_seen": 501088256 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283550651955868, + "loss": 2.7564, + "theoretical_loss": 3.913882635652238, + "tokens_seen": 501153792 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042834503510531596, + "loss": 2.7827, + "theoretical_loss": 3.9138275341533424, + "tokens_seen": 501219328 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042833500501504514, + "loss": 2.7285, + "theoretical_loss": 3.913772441875684, + "tokens_seen": 501284864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 286935, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.952582359313965, + "objective/train/theoretical_loss": 3.9137173588165135, + "objective/train/tokens_used": 521810400, + "theoretical_loss": 3.9137173588165135, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283249749247743, + "loss": 2.888, + "theoretical_loss": 3.9137173588165135, + "tokens_seen": 501350400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004283149448345035, + "loss": 2.7366, + "theoretical_loss": 3.9136622849730833, + "tokens_seen": 501415936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042830491474423274, + "loss": 2.6463, + "theoretical_loss": 3.913607220342648, + "tokens_seen": 501481472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042829488465396186, + "loss": 2.785, + "theoretical_loss": 3.9135521649224634, + "tokens_seen": 501547008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282848545636911, + "loss": 2.9083, + "theoretical_loss": 3.9134971187097847, + "tokens_seen": 501612544 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282748244734203, + "loss": 2.886, + "theoretical_loss": 3.913442081701869, + "tokens_seen": 501678080 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042826479438314946, + "loss": 2.6593, + "theoretical_loss": 3.9133870538959763, + "tokens_seen": 501743616 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042825476429287864, + "loss": 2.8406, + "theoretical_loss": 3.9133320352893657, + "tokens_seen": 501809152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282447342026078, + "loss": 2.881, + "theoretical_loss": 3.913277025879298, + "tokens_seen": 501874688 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428234704112337, + "loss": 2.8322, + "theoretical_loss": 3.9132220256630363, + "tokens_seen": 501940224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042822467402206624, + "loss": 2.919, + "theoretical_loss": 3.9131670346378424, + "tokens_seen": 502005760 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042821464393179537, + "loss": 2.8986, + "theoretical_loss": 3.913112052800982, + "tokens_seen": 502071296 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004282046138415246, + "loss": 3.0142, + "theoretical_loss": 3.9130570801497204, + "tokens_seen": 502136832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042819458375125373, + "loss": 2.8962, + "theoretical_loss": 3.9130021166813247, + "tokens_seen": 502202368 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042818455366098296, + "loss": 2.9193, + "theoretical_loss": 3.9129471623930625, + "tokens_seen": 502267904 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042817452357071215, + "loss": 2.6178, + "theoretical_loss": 3.912892217282203, + "tokens_seen": 502333440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281644934804413, + "loss": 2.7213, + "theoretical_loss": 3.912837281346017, + "tokens_seen": 502398976 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281544633901705, + "loss": 3.0234, + "theoretical_loss": 3.9127823545817755, + "tokens_seen": 502464512 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281444332998997, + "loss": 2.8943, + "theoretical_loss": 3.9127274369867515, + "tokens_seen": 502530048 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042813440320962887, + "loss": 3.3673, + "theoretical_loss": 3.912672528558219, + "tokens_seen": 502595584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004281243731193581, + "loss": 2.9674, + "theoretical_loss": 3.9126176292934525, + "tokens_seen": 502661120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042811434302908723, + "loss": 2.7487, + "theoretical_loss": 3.9125627391897284, + "tokens_seen": 502726656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042810431293881647, + "loss": 2.9542, + "theoretical_loss": 3.9125078582443247, + "tokens_seen": 502792192 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042809428284854565, + "loss": 2.7127, + "theoretical_loss": 3.912452986454519, + "tokens_seen": 502857728 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042808425275827483, + "loss": 2.9038, + "theoretical_loss": 3.9123981238175913, + "tokens_seen": 502923264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 288241, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.815506935119629, + "objective/train/theoretical_loss": 3.9123432703308225, + "objective/train/tokens_used": 523448800, + "theoretical_loss": 3.9123432703308225, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.000428074222668004, + "loss": 2.8686, + "theoretical_loss": 3.9123432703308225, + "tokens_seen": 502988800 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280641925777332, + "loss": 2.8739, + "theoretical_loss": 3.9122884259914947, + "tokens_seen": 503054336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280541624874624, + "loss": 2.9241, + "theoretical_loss": 3.912233590796891, + "tokens_seen": 503119872 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280441323971916, + "loss": 2.8723, + "theoretical_loss": 3.912178764744296, + "tokens_seen": 503185408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042803410230692074, + "loss": 2.7677, + "theoretical_loss": 3.9121239478309944, + "tokens_seen": 503250944 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042802407221664997, + "loss": 2.8224, + "theoretical_loss": 3.9120691400542738, + "tokens_seen": 503316480 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004280140421263791, + "loss": 2.6969, + "theoretical_loss": 3.912014341411421, + "tokens_seen": 503382016 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042800401203610833, + "loss": 2.9163, + "theoretical_loss": 3.9119595518997254, + "tokens_seen": 503447552 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279939819458375, + "loss": 3.1166, + "theoretical_loss": 3.911904771516477, + "tokens_seen": 503513088 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279839518555667, + "loss": 2.8336, + "theoretical_loss": 3.9118500002589682, + "tokens_seen": 503578624 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279739217652959, + "loss": 2.8036, + "theoretical_loss": 3.9117952381244896, + "tokens_seen": 503644160 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279638916750251, + "loss": 2.8712, + "theoretical_loss": 3.911740485110337, + "tokens_seen": 503709696 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042795386158475424, + "loss": 2.8051, + "theoretical_loss": 3.9116857412138026, + "tokens_seen": 503775232 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279438314944835, + "loss": 2.9153, + "theoretical_loss": 3.9116310064321835, + "tokens_seen": 503840768 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279338014042126, + "loss": 2.7278, + "theoretical_loss": 3.9115762807627776, + "tokens_seen": 503906304 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042792377131394184, + "loss": 2.814, + "theoretical_loss": 3.9115215642028813, + "tokens_seen": 503971840 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042791374122367107, + "loss": 2.7999, + "theoretical_loss": 3.911466856749796, + "tokens_seen": 504037376 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004279037111334002, + "loss": 2.8065, + "theoretical_loss": 3.911412158400821, + "tokens_seen": 504102912 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042789368104312943, + "loss": 2.8346, + "theoretical_loss": 3.9113574691532573, + "tokens_seen": 504168448 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042788365095285856, + "loss": 3.0613, + "theoretical_loss": 3.911302789004409, + "tokens_seen": 504233984 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278736208625878, + "loss": 2.9722, + "theoretical_loss": 3.9112481179515797, + "tokens_seen": 504299520 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427863590772317, + "loss": 2.8922, + "theoretical_loss": 3.9111934559920742, + "tokens_seen": 504365056 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042785356068204616, + "loss": 2.7941, + "theoretical_loss": 3.9111388031231993, + "tokens_seen": 504430592 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042784353059177534, + "loss": 2.9636, + "theoretical_loss": 3.911084159342261, + "tokens_seen": 504496128 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278335005015045, + "loss": 2.7491, + "theoretical_loss": 3.91102952464657, + "tokens_seen": 504561664 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 289179, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2568519115448, + "objective/train/theoretical_loss": 3.9109748990334348, + "objective/train/tokens_used": 525087200, + "theoretical_loss": 3.9109748990334348, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004278234704112337, + "loss": 2.8859, + "theoretical_loss": 3.9109748990334348, + "tokens_seen": 504627200 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042781344032096294, + "loss": 2.7951, + "theoretical_loss": 3.910920282500166, + "tokens_seen": 504692736 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042780341023069206, + "loss": 3.0004, + "theoretical_loss": 3.910865675044076, + "tokens_seen": 504758272 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277933801404213, + "loss": 2.8993, + "theoretical_loss": 3.9108110766624775, + "tokens_seen": 504823808 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277833500501505, + "loss": 2.6838, + "theoretical_loss": 3.9107564873526854, + "tokens_seen": 504889344 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042777331995987966, + "loss": 2.6696, + "theoretical_loss": 3.9107019071120153, + "tokens_seen": 504954880 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042776328986960884, + "loss": 2.7526, + "theoretical_loss": 3.910647335937783, + "tokens_seen": 505020416 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427753259779338, + "loss": 3.007, + "theoretical_loss": 3.910592773827306, + "tokens_seen": 505085952 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277432296890672, + "loss": 2.6543, + "theoretical_loss": 3.9105382207779043, + "tokens_seen": 505151488 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042773319959879644, + "loss": 2.9892, + "theoretical_loss": 3.9104836767868973, + "tokens_seen": 505217024 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042772316950852557, + "loss": 2.8852, + "theoretical_loss": 3.9104291418516057, + "tokens_seen": 505282560 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004277131394182548, + "loss": 2.5701, + "theoretical_loss": 3.910374615969352, + "tokens_seen": 505348096 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042770310932798393, + "loss": 3.0667, + "theoretical_loss": 3.9103200991374605, + "tokens_seen": 505413632 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042769307923771316, + "loss": 2.8631, + "theoretical_loss": 3.9102655913532542, + "tokens_seen": 505479168 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042768304914744235, + "loss": 2.7267, + "theoretical_loss": 3.9102110926140603, + "tokens_seen": 505544704 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042767301905717153, + "loss": 2.6952, + "theoretical_loss": 3.910156602917205, + "tokens_seen": 505610240 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276629889669007, + "loss": 2.6144, + "theoretical_loss": 3.9101021222600156, + "tokens_seen": 505675776 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276529588766299, + "loss": 2.9215, + "theoretical_loss": 3.9100476506398216, + "tokens_seen": 505741312 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042764292878635907, + "loss": 3.0363, + "theoretical_loss": 3.909993188053954, + "tokens_seen": 505806848 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004276328986960883, + "loss": 2.9977, + "theoretical_loss": 3.909938734499743, + "tokens_seen": 505872384 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042762286860581743, + "loss": 2.7794, + "theoretical_loss": 3.9098842899745216, + "tokens_seen": 505937920 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042761283851554667, + "loss": 2.673, + "theoretical_loss": 3.909829854475624, + "tokens_seen": 506003456 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042760280842527585, + "loss": 2.8276, + "theoretical_loss": 3.9097754280003842, + "tokens_seen": 506068992 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042759277833500503, + "loss": 2.754, + "theoretical_loss": 3.9097210105461384, + "tokens_seen": 506134528 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275827482447342, + "loss": 3.0903, + "theoretical_loss": 3.9096666021102235, + "tokens_seen": 506200064 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 289852, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1201510429382324, + "objective/train/theoretical_loss": 3.909612202689977, + "objective/train/tokens_used": 526725600, + "theoretical_loss": 3.909612202689977, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275727181544634, + "loss": 2.8416, + "theoretical_loss": 3.909612202689977, + "tokens_seen": 506265600 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275626880641926, + "loss": 2.6884, + "theoretical_loss": 3.90955781228274, + "tokens_seen": 506331136 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275526579739218, + "loss": 2.7749, + "theoretical_loss": 3.909503430885851, + "tokens_seen": 506396672 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042754262788365094, + "loss": 2.8646, + "theoretical_loss": 3.9094490584966524, + "tokens_seen": 506462208 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042753259779338017, + "loss": 2.8293, + "theoretical_loss": 3.9093946951124874, + "tokens_seen": 506527744 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275225677031093, + "loss": 2.9254, + "theoretical_loss": 3.9093403407306986, + "tokens_seen": 506593280 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042751253761283853, + "loss": 2.8896, + "theoretical_loss": 3.9092859953486316, + "tokens_seen": 506658816 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004275025075225677, + "loss": 2.9322, + "theoretical_loss": 3.909231658963633, + "tokens_seen": 506724352 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274924774322969, + "loss": 2.8241, + "theoretical_loss": 3.9091773315730487, + "tokens_seen": 506789888 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274824473420261, + "loss": 2.8595, + "theoretical_loss": 3.909123013174228, + "tokens_seen": 506855424 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274724172517553, + "loss": 3.029, + "theoretical_loss": 3.90906870376452, + "tokens_seen": 506920960 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042746238716148444, + "loss": 2.889, + "theoretical_loss": 3.909014403341275, + "tokens_seen": 506986496 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274523570712137, + "loss": 3.0323, + "theoretical_loss": 3.9089601119018456, + "tokens_seen": 507052032 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274423269809428, + "loss": 3.1528, + "theoretical_loss": 3.908905829443583, + "tokens_seen": 507117568 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042743229689067204, + "loss": 2.9999, + "theoretical_loss": 3.9088515559638424, + "tokens_seen": 507183104 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274222668004012, + "loss": 2.8889, + "theoretical_loss": 3.9087972914599787, + "tokens_seen": 507248640 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274122367101304, + "loss": 2.9263, + "theoretical_loss": 3.908743035929348, + "tokens_seen": 507314176 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004274022066198596, + "loss": 2.697, + "theoretical_loss": 3.9086887893693065, + "tokens_seen": 507379712 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042739217652958876, + "loss": 2.9639, + "theoretical_loss": 3.9086345517772143, + "tokens_seen": 507445248 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042738214643931794, + "loss": 2.8796, + "theoretical_loss": 3.90858032315043, + "tokens_seen": 507510784 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273721163490472, + "loss": 2.7631, + "theoretical_loss": 3.9085261034863144, + "tokens_seen": 507576320 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273620862587763, + "loss": 3.0314, + "theoretical_loss": 3.908471892782229, + "tokens_seen": 507641856 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042735205616850554, + "loss": 2.9613, + "theoretical_loss": 3.9084176910355373, + "tokens_seen": 507707392 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042734202607823467, + "loss": 3.1484, + "theoretical_loss": 3.9083634982436033, + "tokens_seen": 507772928 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273319959879639, + "loss": 2.9547, + "theoretical_loss": 3.908309314403791, + "tokens_seen": 507838464 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 291045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8749454021453857, + "objective/train/theoretical_loss": 3.908255139513467, + "objective/train/tokens_used": 528364000, + "theoretical_loss": 3.908255139513467, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004273219658976931, + "loss": 3.0369, + "theoretical_loss": 3.908255139513467, + "tokens_seen": 507904000 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042731193580742226, + "loss": 2.8929, + "theoretical_loss": 3.9082009735699996, + "tokens_seen": 507969536 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042730190571715145, + "loss": 2.7172, + "theoretical_loss": 3.9081468165707562, + "tokens_seen": 508035072 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272918756268807, + "loss": 2.8316, + "theoretical_loss": 3.9080926685131074, + "tokens_seen": 508100608 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272818455366098, + "loss": 2.7872, + "theoretical_loss": 3.908038529394423, + "tokens_seen": 508166144 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042727181544633904, + "loss": 2.8132, + "theoretical_loss": 3.907984399212075, + "tokens_seen": 508231680 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042726178535606817, + "loss": 2.9742, + "theoretical_loss": 3.9079302779634366, + "tokens_seen": 508297216 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272517552657974, + "loss": 2.7523, + "theoretical_loss": 3.907876165645881, + "tokens_seen": 508362752 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272417251755266, + "loss": 2.9132, + "theoretical_loss": 3.907822062256784, + "tokens_seen": 508428288 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042723169508525577, + "loss": 2.6844, + "theoretical_loss": 3.9077679677935224, + "tokens_seen": 508493824 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042722166499498495, + "loss": 2.5976, + "theoretical_loss": 3.9077138822534723, + "tokens_seen": 508559360 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042721163490471413, + "loss": 2.9316, + "theoretical_loss": 3.9076598056340126, + "tokens_seen": 508624896 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004272016048144433, + "loss": 3.027, + "theoretical_loss": 3.9076057379325233, + "tokens_seen": 508690432 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042719157472417255, + "loss": 2.8114, + "theoretical_loss": 3.9075516791463842, + "tokens_seen": 508755968 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271815446339017, + "loss": 2.8752, + "theoretical_loss": 3.907497629272978, + "tokens_seen": 508821504 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271715145436309, + "loss": 2.7832, + "theoretical_loss": 3.907443588309687, + "tokens_seen": 508887040 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271614844533601, + "loss": 2.9496, + "theoretical_loss": 3.9073895562538956, + "tokens_seen": 508952576 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042715145436308927, + "loss": 2.7524, + "theoretical_loss": 3.907335533102988, + "tokens_seen": 509018112 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004271414242728185, + "loss": 2.7836, + "theoretical_loss": 3.9072815188543517, + "tokens_seen": 509083648 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042713139418254763, + "loss": 2.78, + "theoretical_loss": 3.907227513505373, + "tokens_seen": 509149184 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042712136409227687, + "loss": 3.1176, + "theoretical_loss": 3.9071735170534407, + "tokens_seen": 509214720 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042711133400200605, + "loss": 2.789, + "theoretical_loss": 3.9071195294959438, + "tokens_seen": 509280256 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042710130391173523, + "loss": 2.7216, + "theoretical_loss": 3.9070655508302736, + "tokens_seen": 509345792 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270912738214644, + "loss": 2.9398, + "theoretical_loss": 3.9070115810538217, + "tokens_seen": 509411328 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270812437311936, + "loss": 2.7936, + "theoretical_loss": 3.9069576201639804, + "tokens_seen": 509476864 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 291444, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.291428804397583, + "objective/train/theoretical_loss": 3.9069036681581446, + "objective/train/tokens_used": 530002400, + "theoretical_loss": 3.9069036681581446, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270712136409228, + "loss": 2.9279, + "theoretical_loss": 3.9069036681581446, + "tokens_seen": 509542400 + }, + { + "epoch": 0.15, + "learning_rate": 0.000427061183550652, + "loss": 3.0056, + "theoretical_loss": 3.906849725033708, + "tokens_seen": 509607936 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042705115346038114, + "loss": 2.9895, + "theoretical_loss": 3.906795790788067, + "tokens_seen": 509673472 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042704112337011037, + "loss": 2.8832, + "theoretical_loss": 3.9067418654186197, + "tokens_seen": 509739008 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270310932798395, + "loss": 2.8935, + "theoretical_loss": 3.9066879489227633, + "tokens_seen": 509804544 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042702106318956873, + "loss": 2.7562, + "theoretical_loss": 3.906634041297898, + "tokens_seen": 509870080 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270110330992979, + "loss": 3.0871, + "theoretical_loss": 3.906580142541424, + "tokens_seen": 509935616 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004270010030090271, + "loss": 2.7015, + "theoretical_loss": 3.906526252650743, + "tokens_seen": 510001152 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269909729187563, + "loss": 3.0532, + "theoretical_loss": 3.9064723716232574, + "tokens_seen": 510066688 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269809428284855, + "loss": 2.7929, + "theoretical_loss": 3.9064184994563713, + "tokens_seen": 510132224 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042697091273821464, + "loss": 2.8344, + "theoretical_loss": 3.9063646361474893, + "tokens_seen": 510197760 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269608826479439, + "loss": 2.9352, + "theoretical_loss": 3.906310781694017, + "tokens_seen": 510263296 + }, + { + "epoch": 0.15, + "learning_rate": 0.000426950852557673, + "loss": 2.9629, + "theoretical_loss": 3.9062569360933623, + "tokens_seen": 510328832 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042694082246740224, + "loss": 2.9865, + "theoretical_loss": 3.906203099342933, + "tokens_seen": 510394368 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269307923771314, + "loss": 2.8716, + "theoretical_loss": 3.9061492714401385, + "tokens_seen": 510459904 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269207622868606, + "loss": 2.8544, + "theoretical_loss": 3.9060954523823885, + "tokens_seen": 510525440 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004269107321965898, + "loss": 3.1565, + "theoretical_loss": 3.906041642167095, + "tokens_seen": 510590976 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042690070210631896, + "loss": 2.9906, + "theoretical_loss": 3.9059878407916706, + "tokens_seen": 510656512 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042689067201604814, + "loss": 2.8862, + "theoretical_loss": 3.9059340482535285, + "tokens_seen": 510722048 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268806419257774, + "loss": 2.815, + "theoretical_loss": 3.905880264550083, + "tokens_seen": 510787584 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268706118355065, + "loss": 2.8911, + "theoretical_loss": 3.905826489678751, + "tokens_seen": 510853120 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042686058174523574, + "loss": 2.9307, + "theoretical_loss": 3.9057727236369484, + "tokens_seen": 510918656 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042685055165496487, + "loss": 2.6768, + "theoretical_loss": 3.9057189664220937, + "tokens_seen": 510984192 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268405215646941, + "loss": 2.9439, + "theoretical_loss": 3.905665218031606, + "tokens_seen": 511049728 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268304914744233, + "loss": 2.9301, + "theoretical_loss": 3.9056114784629052, + "tokens_seen": 511115264 + }, + { + "epoch": 0.15, + "objective/train/docs_used": 292775, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.923103094100952, + "objective/train/theoretical_loss": 3.905557747713412, + "objective/train/tokens_used": 531640800, + "theoretical_loss": 3.905557747713412, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042682046138415246, + "loss": 2.9187, + "theoretical_loss": 3.905557747713412, + "tokens_seen": 511180800 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042681043129388165, + "loss": 3.001, + "theoretical_loss": 3.9055040257805493, + "tokens_seen": 511246336 + }, + { + "epoch": 0.15, + "learning_rate": 0.0004268004012036109, + "loss": 2.946, + "theoretical_loss": 3.9054503126617406, + "tokens_seen": 511311872 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042679037111334, + "loss": 2.8961, + "theoretical_loss": 3.90539660835441, + "tokens_seen": 511377408 + }, + { + "epoch": 0.15, + "learning_rate": 0.00042678034102306924, + "loss": 2.9013, + "theoretical_loss": 3.905342912855983, + "tokens_seen": 511442944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042677031093279837, + "loss": 2.8939, + "theoretical_loss": 3.9052892261638865, + "tokens_seen": 511508480 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267602808425276, + "loss": 3.2111, + "theoretical_loss": 3.905235548275548, + "tokens_seen": 511574016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267502507522568, + "loss": 2.9137, + "theoretical_loss": 3.9051818791883965, + "tokens_seen": 511639552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042674022066198597, + "loss": 2.8786, + "theoretical_loss": 3.9051282188998613, + "tokens_seen": 511705088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042673019057171515, + "loss": 2.8007, + "theoretical_loss": 3.9050745674073744, + "tokens_seen": 511770624 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042672016048144433, + "loss": 2.7962, + "theoretical_loss": 3.9050209247083663, + "tokens_seen": 511836160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004267101303911735, + "loss": 2.9068, + "theoretical_loss": 3.9049672908002715, + "tokens_seen": 511901696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042670010030090275, + "loss": 3.034, + "theoretical_loss": 3.9049136656805237, + "tokens_seen": 511967232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266900702106319, + "loss": 2.7779, + "theoretical_loss": 3.904860049346558, + "tokens_seen": 512032768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266800401203611, + "loss": 2.7907, + "theoretical_loss": 3.9048064417958104, + "tokens_seen": 512098304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042667001003009024, + "loss": 3.0229, + "theoretical_loss": 3.9047528430257192, + "tokens_seen": 512163840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042665997993981947, + "loss": 2.7588, + "theoretical_loss": 3.904699253033723, + "tokens_seen": 512229376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042664994984954865, + "loss": 2.9785, + "theoretical_loss": 3.9046456718172595, + "tokens_seen": 512294912 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042663991975927783, + "loss": 2.7284, + "theoretical_loss": 3.9045920993737715, + "tokens_seen": 512360448 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426629889669007, + "loss": 2.9626, + "theoretical_loss": 3.9045385357006994, + "tokens_seen": 512425984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042661985957873625, + "loss": 2.9813, + "theoretical_loss": 3.9044849807954867, + "tokens_seen": 512491520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004266098294884654, + "loss": 2.6292, + "theoretical_loss": 3.904431434655576, + "tokens_seen": 512557056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265997993981946, + "loss": 2.8153, + "theoretical_loss": 3.9043778972784144, + "tokens_seen": 512622592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042658976930792374, + "loss": 2.8765, + "theoretical_loss": 3.904324368661446, + "tokens_seen": 512688128 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426579739217653, + "loss": 2.8445, + "theoretical_loss": 3.904270848802119, + "tokens_seen": 512753664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 293228, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3358237743377686, + "objective/train/theoretical_loss": 3.904217337697881, + "objective/train/tokens_used": 533279200, + "theoretical_loss": 3.904217337697881, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042656970912738216, + "loss": 2.832, + "theoretical_loss": 3.904217337697881, + "tokens_seen": 512819200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042655967903711134, + "loss": 2.9333, + "theoretical_loss": 3.904163835346181, + "tokens_seen": 512884736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265496489468405, + "loss": 3.0451, + "theoretical_loss": 3.9041103417444694, + "tokens_seen": 512950272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265396188565697, + "loss": 2.9155, + "theoretical_loss": 3.9040568568901977, + "tokens_seen": 513015808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265295887662989, + "loss": 2.9664, + "theoretical_loss": 3.9040033807808188, + "tokens_seen": 513081344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004265195586760281, + "loss": 3.024, + "theoretical_loss": 3.9039499134137854, + "tokens_seen": 513146880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042650952858575724, + "loss": 2.8769, + "theoretical_loss": 3.903896454786552, + "tokens_seen": 513212416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264994984954865, + "loss": 2.8886, + "theoretical_loss": 3.903843004896575, + "tokens_seen": 513277952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264894684052156, + "loss": 2.8289, + "theoretical_loss": 3.9037895637413103, + "tokens_seen": 513343488 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042647943831494484, + "loss": 2.8079, + "theoretical_loss": 3.903736131318216, + "tokens_seen": 513409024 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426469408224674, + "loss": 2.8159, + "theoretical_loss": 3.9036827076247516, + "tokens_seen": 513474560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264593781344032, + "loss": 2.9007, + "theoretical_loss": 3.9036292926583753, + "tokens_seen": 513540096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264493480441324, + "loss": 2.8079, + "theoretical_loss": 3.903575886416549, + "tokens_seen": 513605632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004264393179538616, + "loss": 2.7109, + "theoretical_loss": 3.9035224888967344, + "tokens_seen": 513671168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042642928786359075, + "loss": 2.8299, + "theoretical_loss": 3.9034691000963955, + "tokens_seen": 513736704 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042641925777332, + "loss": 2.8933, + "theoretical_loss": 3.9034157200129953, + "tokens_seen": 513802240 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042640922768304916, + "loss": 2.8528, + "theoretical_loss": 3.9033623486439994, + "tokens_seen": 513867776 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042639919759277834, + "loss": 3.0509, + "theoretical_loss": 3.9033089859868744, + "tokens_seen": 513933312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263891675025076, + "loss": 2.8992, + "theoretical_loss": 3.9032556320390874, + "tokens_seen": 513998848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263791374122367, + "loss": 2.7701, + "theoretical_loss": 3.9032022867981055, + "tokens_seen": 514064384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042636910732196594, + "loss": 2.9763, + "theoretical_loss": 3.9031489502614, + "tokens_seen": 514129920 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042635907723169507, + "loss": 2.6713, + "theoretical_loss": 3.9030956224264406, + "tokens_seen": 514195456 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263490471414243, + "loss": 2.7494, + "theoretical_loss": 3.903042303290699, + "tokens_seen": 514260992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263390170511535, + "loss": 2.9207, + "theoretical_loss": 3.9029889928516472, + "tokens_seen": 514326528 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042632898696088266, + "loss": 2.804, + "theoretical_loss": 3.902935691106759, + "tokens_seen": 514392064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 294365, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.787262201309204, + "objective/train/theoretical_loss": 3.90288239805351, + "objective/train/tokens_used": 534917600, + "theoretical_loss": 3.90288239805351, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042631895687061185, + "loss": 3.0647, + "theoretical_loss": 3.90288239805351, + "tokens_seen": 514457600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004263089267803411, + "loss": 2.6791, + "theoretical_loss": 3.902829113689375, + "tokens_seen": 514523136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262988966900702, + "loss": 2.7555, + "theoretical_loss": 3.902775838011831, + "tokens_seen": 514588672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042628886659979944, + "loss": 2.9535, + "theoretical_loss": 3.902722571018356, + "tokens_seen": 514654208 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042627883650952857, + "loss": 2.8113, + "theoretical_loss": 3.902669312706429, + "tokens_seen": 514719744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262688064192578, + "loss": 2.8344, + "theoretical_loss": 3.90261606307353, + "tokens_seen": 514785280 + }, + { + "epoch": 0.16, + "learning_rate": 0.000426258776328987, + "loss": 3.0159, + "theoretical_loss": 3.90256282211714, + "tokens_seen": 514850816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042624874623871617, + "loss": 2.8466, + "theoretical_loss": 3.9025095898347404, + "tokens_seen": 514916352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042623871614844535, + "loss": 2.6467, + "theoretical_loss": 3.9024563662238156, + "tokens_seen": 514981888 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042622868605817453, + "loss": 2.5978, + "theoretical_loss": 3.902403151281848, + "tokens_seen": 515047424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004262186559679037, + "loss": 2.5738, + "theoretical_loss": 3.902349945006325, + "tokens_seen": 515112960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042620862587763295, + "loss": 3.0073, + "theoretical_loss": 3.9022967473947308, + "tokens_seen": 515178496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261985957873621, + "loss": 2.8529, + "theoretical_loss": 3.9022435584445545, + "tokens_seen": 515244032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261885656970913, + "loss": 2.8852, + "theoretical_loss": 3.902190378153283, + "tokens_seen": 515309568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042617853560682044, + "loss": 2.8072, + "theoretical_loss": 3.9021372065184066, + "tokens_seen": 515375104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042616850551654967, + "loss": 2.724, + "theoretical_loss": 3.902084043537416, + "tokens_seen": 515440640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042615847542627885, + "loss": 2.8464, + "theoretical_loss": 3.902030889207802, + "tokens_seen": 515506176 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042614844533600803, + "loss": 2.5744, + "theoretical_loss": 3.9019777435270573, + "tokens_seen": 515571712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261384152457372, + "loss": 2.8335, + "theoretical_loss": 3.901924606492675, + "tokens_seen": 515637248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042612838515546645, + "loss": 3.0876, + "theoretical_loss": 3.9018714781021515, + "tokens_seen": 515702784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261183550651956, + "loss": 2.8044, + "theoretical_loss": 3.9018183583529806, + "tokens_seen": 515768320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004261083249749248, + "loss": 2.9733, + "theoretical_loss": 3.9017652472426603, + "tokens_seen": 515833856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042609829488465394, + "loss": 2.9147, + "theoretical_loss": 3.9017121447686876, + "tokens_seen": 515899392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260882647943832, + "loss": 2.93, + "theoretical_loss": 3.9016590509285614, + "tokens_seen": 515964928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042607823470411236, + "loss": 2.8142, + "theoretical_loss": 3.901605965719782, + "tokens_seen": 516030464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 295169, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.053830623626709, + "objective/train/theoretical_loss": 3.9015528891398494, + "objective/train/tokens_used": 536556000, + "theoretical_loss": 3.9015528891398494, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042606820461384154, + "loss": 3.1288, + "theoretical_loss": 3.9015528891398494, + "tokens_seen": 516096000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260581745235707, + "loss": 2.694, + "theoretical_loss": 3.9014998211862673, + "tokens_seen": 516161536 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260481444332999, + "loss": 3.0028, + "theoretical_loss": 3.901446761856537, + "tokens_seen": 516227072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260381143430291, + "loss": 2.7803, + "theoretical_loss": 3.901393711148163, + "tokens_seen": 516292608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260280842527583, + "loss": 2.8191, + "theoretical_loss": 3.901340669058651, + "tokens_seen": 516358144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042601805416248744, + "loss": 3.0177, + "theoretical_loss": 3.9012876355855064, + "tokens_seen": 516423680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004260080240722167, + "loss": 2.7097, + "theoretical_loss": 3.9012346107262363, + "tokens_seen": 516489216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259979939819458, + "loss": 2.9959, + "theoretical_loss": 3.9011815944783494, + "tokens_seen": 516554752 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042598796389167504, + "loss": 3.0318, + "theoretical_loss": 3.9011285868393544, + "tokens_seen": 516620288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259779338014042, + "loss": 2.9474, + "theoretical_loss": 3.901075587806762, + "tokens_seen": 516685824 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259679037111334, + "loss": 3.0822, + "theoretical_loss": 3.901022597378083, + "tokens_seen": 516751360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259578736208626, + "loss": 2.8803, + "theoretical_loss": 3.90096961555083, + "tokens_seen": 516816896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259478435305918, + "loss": 2.8745, + "theoretical_loss": 3.9009166423225166, + "tokens_seen": 516882432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042593781344032095, + "loss": 3.0311, + "theoretical_loss": 3.9008636776906567, + "tokens_seen": 516947968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259277833500502, + "loss": 2.9673, + "theoretical_loss": 3.9008107216527663, + "tokens_seen": 517013504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004259177532597793, + "loss": 2.9108, + "theoretical_loss": 3.9007577742063613, + "tokens_seen": 517079040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042590772316950854, + "loss": 2.7388, + "theoretical_loss": 3.900704835348959, + "tokens_seen": 517144576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258976930792377, + "loss": 2.991, + "theoretical_loss": 3.900651905078079, + "tokens_seen": 517210112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258876629889669, + "loss": 2.7754, + "theoretical_loss": 3.90059898339124, + "tokens_seen": 517275648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258776328986961, + "loss": 2.8331, + "theoretical_loss": 3.9005460702859627, + "tokens_seen": 517341184 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042586760280842527, + "loss": 2.9162, + "theoretical_loss": 3.9004931657597686, + "tokens_seen": 517406720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042585757271815445, + "loss": 2.8646, + "theoretical_loss": 3.900440269810181, + "tokens_seen": 517472256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258475426278837, + "loss": 3.0237, + "theoretical_loss": 3.900387382434723, + "tokens_seen": 517537792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258375125376128, + "loss": 2.8647, + "theoretical_loss": 3.900334503630919, + "tokens_seen": 517603328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042582748244734205, + "loss": 2.8821, + "theoretical_loss": 3.9002816333962955, + "tokens_seen": 517668864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 295690, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6655638217926025, + "objective/train/theoretical_loss": 3.9002287717283783, + "objective/train/tokens_used": 538194400, + "theoretical_loss": 3.9002287717283783, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258174523570712, + "loss": 2.6781, + "theoretical_loss": 3.9002287717283783, + "tokens_seen": 517734400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004258074222668004, + "loss": 2.737, + "theoretical_loss": 3.9001759186246967, + "tokens_seen": 517799936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257973921765296, + "loss": 2.8423, + "theoretical_loss": 3.900123074082778, + "tokens_seen": 517865472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042578736208625877, + "loss": 3.0461, + "theoretical_loss": 3.9000702381001533, + "tokens_seen": 517931008 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042577733199598795, + "loss": 3.0473, + "theoretical_loss": 3.900017410674352, + "tokens_seen": 517996544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257673019057172, + "loss": 2.764, + "theoretical_loss": 3.899964591802907, + "tokens_seen": 518062080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257572718154463, + "loss": 3.0369, + "theoretical_loss": 3.8999117814833513, + "tokens_seen": 518127616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042574724172517555, + "loss": 2.8756, + "theoretical_loss": 3.899858979713218, + "tokens_seen": 518193152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257372116349047, + "loss": 3.0317, + "theoretical_loss": 3.899806186490043, + "tokens_seen": 518258688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257271815446339, + "loss": 2.885, + "theoretical_loss": 3.8997534018113615, + "tokens_seen": 518324224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257171514543631, + "loss": 2.8826, + "theoretical_loss": 3.8997006256747113, + "tokens_seen": 518389760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004257071213640923, + "loss": 2.9002, + "theoretical_loss": 3.8996478580776293, + "tokens_seen": 518455296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042569709127382145, + "loss": 2.8571, + "theoretical_loss": 3.899595099017656, + "tokens_seen": 518520832 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042568706118355064, + "loss": 2.8265, + "theoretical_loss": 3.8995423484923304, + "tokens_seen": 518586368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256770310932798, + "loss": 2.7872, + "theoretical_loss": 3.899489606499193, + "tokens_seen": 518651904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042566700100300905, + "loss": 2.8207, + "theoretical_loss": 3.8994368730357873, + "tokens_seen": 518717440 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042565697091273823, + "loss": 2.8399, + "theoretical_loss": 3.8993841480996565, + "tokens_seen": 518782976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256469408224674, + "loss": 2.8967, + "theoretical_loss": 3.899331431688344, + "tokens_seen": 518848512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042563691073219665, + "loss": 2.8206, + "theoretical_loss": 3.8992787237993944, + "tokens_seen": 518914048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004256268806419258, + "loss": 2.6858, + "theoretical_loss": 3.8992260244303547, + "tokens_seen": 518979584 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425616850551655, + "loss": 2.8494, + "theoretical_loss": 3.899173333578772, + "tokens_seen": 519045120 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042560682046138414, + "loss": 2.7067, + "theoretical_loss": 3.8991206512421943, + "tokens_seen": 519110656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255967903711134, + "loss": 2.8115, + "theoretical_loss": 3.8990679774181713, + "tokens_seen": 519176192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042558676028084256, + "loss": 2.8671, + "theoretical_loss": 3.899015312104253, + "tokens_seen": 519241728 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042557673019057174, + "loss": 2.787, + "theoretical_loss": 3.89896265529799, + "tokens_seen": 519307264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 296800, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2833645343780518, + "objective/train/theoretical_loss": 3.8989100069969354, + "objective/train/tokens_used": 539832800, + "theoretical_loss": 3.8989100069969354, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255667001003009, + "loss": 2.8438, + "theoretical_loss": 3.8989100069969354, + "tokens_seen": 519372800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255566700100301, + "loss": 2.7813, + "theoretical_loss": 3.8988573671986417, + "tokens_seen": 519438336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255466399197593, + "loss": 2.6913, + "theoretical_loss": 3.898804735900664, + "tokens_seen": 519503872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255366098294885, + "loss": 2.9325, + "theoretical_loss": 3.8987521131005574, + "tokens_seen": 519569408 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042552657973921764, + "loss": 3.0525, + "theoretical_loss": 3.898699498795878, + "tokens_seen": 519634944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004255165496489469, + "loss": 3.1233, + "theoretical_loss": 3.898646892984183, + "tokens_seen": 519700480 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425506519558676, + "loss": 2.9536, + "theoretical_loss": 3.8985942956630306, + "tokens_seen": 519766016 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042549648946840524, + "loss": 2.8703, + "theoretical_loss": 3.898541706829981, + "tokens_seen": 519831552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254864593781344, + "loss": 3.0978, + "theoretical_loss": 3.898489126482594, + "tokens_seen": 519897088 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254764292878636, + "loss": 3.0565, + "theoretical_loss": 3.8984365546184305, + "tokens_seen": 519962624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254663991975928, + "loss": 3.0101, + "theoretical_loss": 3.898383991235053, + "tokens_seen": 520028160 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425456369107322, + "loss": 3.0552, + "theoretical_loss": 3.898331436330026, + "tokens_seen": 520093696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042544633901705115, + "loss": 2.6484, + "theoretical_loss": 3.8982788899009124, + "tokens_seen": 520159232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254363089267804, + "loss": 2.9835, + "theoretical_loss": 3.8982263519452793, + "tokens_seen": 520224768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254262788365095, + "loss": 3.0324, + "theoretical_loss": 3.8981738224606914, + "tokens_seen": 520290304 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042541624874623874, + "loss": 2.9913, + "theoretical_loss": 3.898121301444716, + "tokens_seen": 520355840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004254062186559679, + "loss": 2.8293, + "theoretical_loss": 3.898068788894923, + "tokens_seen": 520421376 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253961885656971, + "loss": 3.0361, + "theoretical_loss": 3.898016284808881, + "tokens_seen": 520486912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253861584754263, + "loss": 2.9718, + "theoretical_loss": 3.8979637891841605, + "tokens_seen": 520552448 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042537612838515547, + "loss": 2.8781, + "theoretical_loss": 3.8979113020183327, + "tokens_seen": 520617984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042536609829488465, + "loss": 2.9975, + "theoretical_loss": 3.8978588233089706, + "tokens_seen": 520683520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253560682046139, + "loss": 2.7801, + "theoretical_loss": 3.897806353053647, + "tokens_seen": 520749056 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425346038114343, + "loss": 2.9144, + "theoretical_loss": 3.897753891249936, + "tokens_seen": 520814592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042533600802407225, + "loss": 3.0736, + "theoretical_loss": 3.897701437895414, + "tokens_seen": 520880128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253259779338014, + "loss": 2.9123, + "theoretical_loss": 3.8976489929876568, + "tokens_seen": 520945664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 297883, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9455604553222656, + "objective/train/theoretical_loss": 3.8975965565242423, + "objective/train/tokens_used": 541471200, + "theoretical_loss": 3.8975965565242423, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253159478435306, + "loss": 2.724, + "theoretical_loss": 3.8975965565242423, + "tokens_seen": 521011200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004253059177532598, + "loss": 3.0535, + "theoretical_loss": 3.897544128502749, + "tokens_seen": 521076736 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042529588766298897, + "loss": 2.9999, + "theoretical_loss": 3.897491708920755, + "tokens_seen": 521142272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042528585757271815, + "loss": 2.9009, + "theoretical_loss": 3.897439297775842, + "tokens_seen": 521207808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252758274824474, + "loss": 2.8293, + "theoretical_loss": 3.8973868950655914, + "tokens_seen": 521273344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252657973921765, + "loss": 2.9852, + "theoretical_loss": 3.8973345007875846, + "tokens_seen": 521338880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042525576730190575, + "loss": 2.9392, + "theoretical_loss": 3.897282114939406, + "tokens_seen": 521404416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252457372116349, + "loss": 2.8227, + "theoretical_loss": 3.89722973751864, + "tokens_seen": 521469952 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252357071213641, + "loss": 2.7058, + "theoretical_loss": 3.897177368522872, + "tokens_seen": 521535488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252256770310933, + "loss": 2.9889, + "theoretical_loss": 3.897125007949687, + "tokens_seen": 521601024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004252156469408225, + "loss": 2.935, + "theoretical_loss": 3.8970726557966744, + "tokens_seen": 521666560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042520561685055165, + "loss": 2.9667, + "theoretical_loss": 3.897020312061421, + "tokens_seen": 521732096 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042519558676028084, + "loss": 2.941, + "theoretical_loss": 3.896967976741517, + "tokens_seen": 521797632 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042518555667001, + "loss": 2.9092, + "theoretical_loss": 3.896915649834553, + "tokens_seen": 521863168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042517552657973925, + "loss": 2.6848, + "theoretical_loss": 3.8968633313381202, + "tokens_seen": 521928704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251654964894684, + "loss": 2.7817, + "theoretical_loss": 3.8968110212498104, + "tokens_seen": 521994240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251554663991976, + "loss": 2.7736, + "theoretical_loss": 3.8967587195672175, + "tokens_seen": 522059776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251454363089268, + "loss": 2.9141, + "theoretical_loss": 3.8967064262879356, + "tokens_seen": 522125312 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425135406218656, + "loss": 2.6049, + "theoretical_loss": 3.8966541414095595, + "tokens_seen": 522190848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042512537612838516, + "loss": 2.6158, + "theoretical_loss": 3.8966018649296865, + "tokens_seen": 522256384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042511534603811434, + "loss": 2.7402, + "theoretical_loss": 3.8965495968459134, + "tokens_seen": 522321920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004251053159478435, + "loss": 3.1019, + "theoretical_loss": 3.896497337155839, + "tokens_seen": 522387456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042509528585757276, + "loss": 2.9382, + "theoretical_loss": 3.896445085857062, + "tokens_seen": 522452992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250852557673019, + "loss": 2.9426, + "theoretical_loss": 3.896392842947183, + "tokens_seen": 522518528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250752256770311, + "loss": 2.6713, + "theoretical_loss": 3.8963406084238024, + "tokens_seen": 522584064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 298609, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9465911388397217, + "objective/train/theoretical_loss": 3.8962883822845233, + "objective/train/tokens_used": 543109600, + "theoretical_loss": 3.8962883822845233, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042506519558676024, + "loss": 2.7065, + "theoretical_loss": 3.8962883822845233, + "tokens_seen": 522649600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250551654964895, + "loss": 2.9139, + "theoretical_loss": 3.8962361645269494, + "tokens_seen": 522715136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042504513540621866, + "loss": 2.9572, + "theoretical_loss": 3.8961839551486843, + "tokens_seen": 522780672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042503510531594784, + "loss": 2.8489, + "theoretical_loss": 3.8961317541473326, + "tokens_seen": 522846208 + }, + { + "epoch": 0.16, + "learning_rate": 0.000425025075225677, + "loss": 3.0263, + "theoretical_loss": 3.8960795615205015, + "tokens_seen": 522911744 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250150451354062, + "loss": 3.0704, + "theoretical_loss": 3.896027377265798, + "tokens_seen": 522977280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004250050150451354, + "loss": 2.9052, + "theoretical_loss": 3.8959752013808293, + "tokens_seen": 523042816 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249949849548646, + "loss": 2.916, + "theoretical_loss": 3.895923033863206, + "tokens_seen": 523108352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042498495486459375, + "loss": 2.955, + "theoretical_loss": 3.8958708747105373, + "tokens_seen": 523173888 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424974924774323, + "loss": 2.8302, + "theoretical_loss": 3.895818723920434, + "tokens_seen": 523239424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042496489468405216, + "loss": 2.7975, + "theoretical_loss": 3.895766581490509, + "tokens_seen": 523304960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042495486459378135, + "loss": 2.8087, + "theoretical_loss": 3.895714447418375, + "tokens_seen": 523370496 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249448345035105, + "loss": 2.9109, + "theoretical_loss": 3.8956623217016464, + "tokens_seen": 523436032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249348044132397, + "loss": 2.8087, + "theoretical_loss": 3.895610204337938, + "tokens_seen": 523501568 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042492477432296894, + "loss": 2.8582, + "theoretical_loss": 3.8955580953248656, + "tokens_seen": 523567104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249147442326981, + "loss": 2.8843, + "theoretical_loss": 3.895505994660046, + "tokens_seen": 523632640 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004249047141424273, + "loss": 3.1563, + "theoretical_loss": 3.8954539023410972, + "tokens_seen": 523698176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248946840521565, + "loss": 2.9491, + "theoretical_loss": 3.895401818365639, + "tokens_seen": 523763712 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042488465396188567, + "loss": 2.9519, + "theoretical_loss": 3.89534974273129, + "tokens_seen": 523829248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042487462387161485, + "loss": 2.5973, + "theoretical_loss": 3.895297675435673, + "tokens_seen": 523894784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248645937813441, + "loss": 2.8273, + "theoretical_loss": 3.8952456164764078, + "tokens_seen": 523960320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248545636910732, + "loss": 2.7711, + "theoretical_loss": 3.8951935658511183, + "tokens_seen": 524025856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042484453360080245, + "loss": 2.9623, + "theoretical_loss": 3.895141523557428, + "tokens_seen": 524091392 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248345035105316, + "loss": 2.7556, + "theoretical_loss": 3.895089489592962, + "tokens_seen": 524156928 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004248244734202608, + "loss": 2.907, + "theoretical_loss": 3.8950374639553456, + "tokens_seen": 524222464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 299683, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.230189085006714, + "objective/train/theoretical_loss": 3.894985446642206, + "objective/train/tokens_used": 544748000, + "theoretical_loss": 3.894985446642206, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042481444332999, + "loss": 2.674, + "theoretical_loss": 3.894985446642206, + "tokens_seen": 524288000 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042480441323971917, + "loss": 2.7178, + "theoretical_loss": 3.8949334376511713, + "tokens_seen": 524353536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042479438314944835, + "loss": 2.6602, + "theoretical_loss": 3.894881436979869, + "tokens_seen": 524419072 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247843530591776, + "loss": 2.881, + "theoretical_loss": 3.8948294446259295, + "tokens_seen": 524484608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247743229689067, + "loss": 3.029, + "theoretical_loss": 3.894777460586983, + "tokens_seen": 524550144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042476429287863595, + "loss": 3.0115, + "theoretical_loss": 3.894725484860662, + "tokens_seen": 524615680 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247542627883651, + "loss": 2.9568, + "theoretical_loss": 3.8946735174445983, + "tokens_seen": 524681216 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247442326980943, + "loss": 2.6509, + "theoretical_loss": 3.894621558336426, + "tokens_seen": 524746752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247342026078235, + "loss": 2.7833, + "theoretical_loss": 3.8945696075337786, + "tokens_seen": 524812288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004247241725175527, + "loss": 2.8974, + "theoretical_loss": 3.8945176650342925, + "tokens_seen": 524877824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042471414242728186, + "loss": 3.0308, + "theoretical_loss": 3.894465730835604, + "tokens_seen": 524943360 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042470411233701104, + "loss": 2.8243, + "theoretical_loss": 3.8944138049353505, + "tokens_seen": 525008896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246940822467402, + "loss": 2.8344, + "theoretical_loss": 3.8943618873311703, + "tokens_seen": 525074432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042468405215646945, + "loss": 2.6421, + "theoretical_loss": 3.8943099780207024, + "tokens_seen": 525139968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246740220661986, + "loss": 3.2505, + "theoretical_loss": 3.894258077001587, + "tokens_seen": 525205504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246639919759278, + "loss": 2.9114, + "theoretical_loss": 3.8942061842714666, + "tokens_seen": 525271040 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424653961885657, + "loss": 2.9455, + "theoretical_loss": 3.8941542998279823, + "tokens_seen": 525336576 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246439317953862, + "loss": 2.7369, + "theoretical_loss": 3.894102423668778, + "tokens_seen": 525402112 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042463390170511536, + "loss": 2.879, + "theoretical_loss": 3.8940505557914973, + "tokens_seen": 525467648 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042462387161484454, + "loss": 2.9338, + "theoretical_loss": 3.893998696193785, + "tokens_seen": 525533184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004246138415245737, + "loss": 2.7185, + "theoretical_loss": 3.8939468448732892, + "tokens_seen": 525598720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042460381143430296, + "loss": 2.723, + "theoretical_loss": 3.893895001827654, + "tokens_seen": 525664256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245937813440321, + "loss": 2.8829, + "theoretical_loss": 3.89384316705453, + "tokens_seen": 525729792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245837512537613, + "loss": 2.8745, + "theoretical_loss": 3.893791340551565, + "tokens_seen": 525795328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042457372116349044, + "loss": 2.9526, + "theoretical_loss": 3.8937395223164093, + "tokens_seen": 525860864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 300593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.640450954437256, + "objective/train/theoretical_loss": 3.893687712346713, + "objective/train/tokens_used": 546386400, + "theoretical_loss": 3.893687712346713, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245636910732197, + "loss": 3.066, + "theoretical_loss": 3.893687712346713, + "tokens_seen": 525926400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042455366098294886, + "loss": 2.814, + "theoretical_loss": 3.8936359106401293, + "tokens_seen": 525991936 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042454363089267804, + "loss": 2.7216, + "theoretical_loss": 3.8935841171943095, + "tokens_seen": 526057472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245336008024072, + "loss": 2.7412, + "theoretical_loss": 3.893532332006909, + "tokens_seen": 526123008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245235707121364, + "loss": 2.9448, + "theoretical_loss": 3.8934805550755813, + "tokens_seen": 526188544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245135406218656, + "loss": 2.7285, + "theoretical_loss": 3.893428786397983, + "tokens_seen": 526254080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004245035105315948, + "loss": 2.7658, + "theoretical_loss": 3.8933770259717697, + "tokens_seen": 526319616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042449348044132395, + "loss": 3.0131, + "theoretical_loss": 3.8933252737946002, + "tokens_seen": 526385152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244834503510532, + "loss": 2.7621, + "theoretical_loss": 3.893273529864132, + "tokens_seen": 526450688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042447342026078236, + "loss": 2.893, + "theoretical_loss": 3.893221794178025, + "tokens_seen": 526516224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042446339017051155, + "loss": 2.8413, + "theoretical_loss": 3.893170066733941, + "tokens_seen": 526581760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244533600802407, + "loss": 2.6966, + "theoretical_loss": 3.8931183475295397, + "tokens_seen": 526647296 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244433299899699, + "loss": 2.6659, + "theoretical_loss": 3.8930666365624838, + "tokens_seen": 526712832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244332998996991, + "loss": 2.8686, + "theoretical_loss": 3.893014933830437, + "tokens_seen": 526778368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244232698094283, + "loss": 2.7931, + "theoretical_loss": 3.8929632393310634, + "tokens_seen": 526843904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042441323971915745, + "loss": 2.9316, + "theoretical_loss": 3.892911553062029, + "tokens_seen": 526909440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004244032096288867, + "loss": 2.7593, + "theoretical_loss": 3.892859875020999, + "tokens_seen": 526974976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243931795386158, + "loss": 2.6133, + "theoretical_loss": 3.8928082052056405, + "tokens_seen": 527040512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042438314944834505, + "loss": 3.0805, + "theoretical_loss": 3.892756543613623, + "tokens_seen": 527106048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042437311935807423, + "loss": 3.0741, + "theoretical_loss": 3.892704890242614, + "tokens_seen": 527171584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243630892678034, + "loss": 2.9281, + "theoretical_loss": 3.892653245090284, + "tokens_seen": 527237120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243530591775326, + "loss": 2.9436, + "theoretical_loss": 3.8926016081543047, + "tokens_seen": 527302656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243430290872618, + "loss": 2.8254, + "theoretical_loss": 3.892549979432347, + "tokens_seen": 527368192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042433299899699095, + "loss": 2.8296, + "theoretical_loss": 3.8924983589220843, + "tokens_seen": 527433728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243229689067202, + "loss": 2.8305, + "theoretical_loss": 3.8924467466211903, + "tokens_seen": 527499264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 301147, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.089402198791504, + "objective/train/theoretical_loss": 3.8923951425273398, + "objective/train/tokens_used": 548024800, + "theoretical_loss": 3.8923951425273398, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004243129388164493, + "loss": 2.883, + "theoretical_loss": 3.8923951425273398, + "tokens_seen": 527564800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042430290872617855, + "loss": 2.8607, + "theoretical_loss": 3.8923435466382084, + "tokens_seen": 527630336 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042429287863590773, + "loss": 2.9292, + "theoretical_loss": 3.892291958951473, + "tokens_seen": 527695872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242828485456369, + "loss": 2.8227, + "theoretical_loss": 3.892240379464811, + "tokens_seen": 527761408 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242728184553661, + "loss": 2.941, + "theoretical_loss": 3.8921888081759004, + "tokens_seen": 527826944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242627883650953, + "loss": 2.7021, + "theoretical_loss": 3.892137245082422, + "tokens_seen": 527892480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042425275827482446, + "loss": 2.98, + "theoretical_loss": 3.8920856901820553, + "tokens_seen": 527958016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242427281845537, + "loss": 2.8898, + "theoretical_loss": 3.8920341434724817, + "tokens_seen": 528023552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242326980942828, + "loss": 2.8186, + "theoretical_loss": 3.8919826049513837, + "tokens_seen": 528089088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042422266800401206, + "loss": 2.9792, + "theoretical_loss": 3.891931074616444, + "tokens_seen": 528154624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242126379137412, + "loss": 3.1397, + "theoretical_loss": 3.8918795524653484, + "tokens_seen": 528220160 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004242026078234704, + "loss": 2.5991, + "theoretical_loss": 3.89182803849578, + "tokens_seen": 528285696 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241925777331996, + "loss": 2.9587, + "theoretical_loss": 3.8917765327054266, + "tokens_seen": 528351232 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241825476429288, + "loss": 2.8182, + "theoretical_loss": 3.8917250350919748, + "tokens_seen": 528416768 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424172517552658, + "loss": 2.8705, + "theoretical_loss": 3.891673545653112, + "tokens_seen": 528482304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241624874623872, + "loss": 2.6681, + "theoretical_loss": 3.891622064386527, + "tokens_seen": 528547840 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241524573721164, + "loss": 2.9959, + "theoretical_loss": 3.89157059128991, + "tokens_seen": 528613376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042414242728184556, + "loss": 2.9152, + "theoretical_loss": 3.8915191263609525, + "tokens_seen": 528678912 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042413239719157474, + "loss": 2.9309, + "theoretical_loss": 3.8914676695973456, + "tokens_seen": 528744448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241223671013039, + "loss": 2.8499, + "theoretical_loss": 3.8914162209967813, + "tokens_seen": 528809984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042411233701103316, + "loss": 2.915, + "theoretical_loss": 3.8913647805569545, + "tokens_seen": 528875520 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004241023069207623, + "loss": 2.9174, + "theoretical_loss": 3.891313348275559, + "tokens_seen": 528941056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240922768304915, + "loss": 2.9057, + "theoretical_loss": 3.8912619241502906, + "tokens_seen": 529006592 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042408224674022065, + "loss": 2.674, + "theoretical_loss": 3.891210508178845, + "tokens_seen": 529072128 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240722166499499, + "loss": 2.7977, + "theoretical_loss": 3.8911591003589203, + "tokens_seen": 529137664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 301749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2091262340545654, + "objective/train/theoretical_loss": 3.891107700688215, + "objective/train/tokens_used": 549663200, + "theoretical_loss": 3.891107700688215, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042406218655967906, + "loss": 2.9569, + "theoretical_loss": 3.891107700688215, + "tokens_seen": 529203200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042405215646940824, + "loss": 2.9637, + "theoretical_loss": 3.8910563091644277, + "tokens_seen": 529268736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240421263791374, + "loss": 2.8253, + "theoretical_loss": 3.891004925785259, + "tokens_seen": 529334272 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240320962888666, + "loss": 2.7514, + "theoretical_loss": 3.890953550548409, + "tokens_seen": 529399808 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004240220661985958, + "loss": 2.8764, + "theoretical_loss": 3.8909021834515816, + "tokens_seen": 529465344 + }, + { + "epoch": 0.16, + "learning_rate": 0.000424012036108325, + "loss": 2.8901, + "theoretical_loss": 3.8908508244924778, + "tokens_seen": 529530880 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042400200601805415, + "loss": 2.7382, + "theoretical_loss": 3.8907994736688027, + "tokens_seen": 529596416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239919759277834, + "loss": 2.7764, + "theoretical_loss": 3.8907481309782606, + "tokens_seen": 529661952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042398194583751256, + "loss": 3.0124, + "theoretical_loss": 3.890696796418558, + "tokens_seen": 529727488 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042397191574724175, + "loss": 2.912, + "theoretical_loss": 3.8906454699874002, + "tokens_seen": 529793024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239618856569709, + "loss": 2.8759, + "theoretical_loss": 3.890594151682496, + "tokens_seen": 529858560 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239518555667001, + "loss": 2.779, + "theoretical_loss": 3.8905428415015537, + "tokens_seen": 529924096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239418254764293, + "loss": 2.8285, + "theoretical_loss": 3.8904915394422828, + "tokens_seen": 529989632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239317953861585, + "loss": 2.7266, + "theoretical_loss": 3.8904402455023934, + "tokens_seen": 530055168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042392176529588765, + "loss": 2.9637, + "theoretical_loss": 3.890388959679597, + "tokens_seen": 530120704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004239117352056169, + "loss": 2.7824, + "theoretical_loss": 3.8903376819716056, + "tokens_seen": 530186240 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423901705115346, + "loss": 2.9065, + "theoretical_loss": 3.890286412376133, + "tokens_seen": 530251776 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042389167502507525, + "loss": 2.7732, + "theoretical_loss": 3.8902351508908923, + "tokens_seen": 530317312 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042388164493480443, + "loss": 2.7168, + "theoretical_loss": 3.8901838975136, + "tokens_seen": 530382848 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238716148445336, + "loss": 2.9481, + "theoretical_loss": 3.890132652241971, + "tokens_seen": 530448384 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238615847542628, + "loss": 2.4908, + "theoretical_loss": 3.890081415073723, + "tokens_seen": 530513920 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423851554663992, + "loss": 3.0909, + "theoretical_loss": 3.8900301860065727, + "tokens_seen": 530579456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042384152457372115, + "loss": 2.7948, + "theoretical_loss": 3.8899789650382397, + "tokens_seen": 530644992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238314944834504, + "loss": 2.9124, + "theoretical_loss": 3.889927752166443, + "tokens_seen": 530710528 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004238214643931795, + "loss": 2.7639, + "theoretical_loss": 3.889876547388904, + "tokens_seen": 530776064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 302930, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.119241714477539, + "objective/train/theoretical_loss": 3.889825350703344, + "objective/train/tokens_used": 551301600, + "theoretical_loss": 3.889825350703344, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042381143430290875, + "loss": 2.8456, + "theoretical_loss": 3.889825350703344, + "tokens_seen": 530841600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042380140421263793, + "loss": 3.1018, + "theoretical_loss": 3.889774162107485, + "tokens_seen": 530907136 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237913741223671, + "loss": 2.637, + "theoretical_loss": 3.889722981599051, + "tokens_seen": 530972672 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237813440320963, + "loss": 2.8671, + "theoretical_loss": 3.8896718091757654, + "tokens_seen": 531038208 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237713139418255, + "loss": 2.8487, + "theoretical_loss": 3.889620644835354, + "tokens_seen": 531103744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042376128385155466, + "loss": 2.7057, + "theoretical_loss": 3.889569488575543, + "tokens_seen": 531169280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237512537612839, + "loss": 2.6746, + "theoretical_loss": 3.8895183403940594, + "tokens_seen": 531234816 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423741223671013, + "loss": 2.564, + "theoretical_loss": 3.889467200288631, + "tokens_seen": 531300352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042373119358074226, + "loss": 2.684, + "theoretical_loss": 3.889416068256986, + "tokens_seen": 531365888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237211634904714, + "loss": 2.7755, + "theoretical_loss": 3.8893649442968554, + "tokens_seen": 531431424 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237111334002006, + "loss": 2.9413, + "theoretical_loss": 3.8893138284059696, + "tokens_seen": 531496960 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004237011033099298, + "loss": 3.0095, + "theoretical_loss": 3.8892627205820602, + "tokens_seen": 531562496 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423691073219659, + "loss": 2.9443, + "theoretical_loss": 3.8892116208228593, + "tokens_seen": 531628032 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042368104312938816, + "loss": 2.7001, + "theoretical_loss": 3.8891605291261007, + "tokens_seen": 531693568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236710130391174, + "loss": 2.7396, + "theoretical_loss": 3.889109445489519, + "tokens_seen": 531759104 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236609829488465, + "loss": 2.8817, + "theoretical_loss": 3.8890583699108494, + "tokens_seen": 531824640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042365095285857576, + "loss": 3.0576, + "theoretical_loss": 3.889007302387828, + "tokens_seen": 531890176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236409227683049, + "loss": 2.9742, + "theoretical_loss": 3.8889562429181916, + "tokens_seen": 531955712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236308926780341, + "loss": 2.8414, + "theoretical_loss": 3.8889051914996786, + "tokens_seen": 532021248 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236208625877633, + "loss": 2.9494, + "theoretical_loss": 3.888854148130028, + "tokens_seen": 532086784 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004236108324974925, + "loss": 2.9868, + "theoretical_loss": 3.88880311280698, + "tokens_seen": 532152320 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042360080240722166, + "loss": 2.7983, + "theoretical_loss": 3.8887520855282744, + "tokens_seen": 532217856 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042359077231695085, + "loss": 2.9949, + "theoretical_loss": 3.888701066291654, + "tokens_seen": 532283392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042358074222668, + "loss": 2.7991, + "theoretical_loss": 3.888650055094861, + "tokens_seen": 532348928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042357071213640926, + "loss": 2.8885, + "theoretical_loss": 3.888599051935638, + "tokens_seen": 532414464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 303466, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.733795404434204, + "objective/train/theoretical_loss": 3.8885480568117314, + "objective/train/tokens_used": 552940000, + "theoretical_loss": 3.8885480568117314, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235606820461384, + "loss": 2.8752, + "theoretical_loss": 3.8885480568117314, + "tokens_seen": 532480000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004235506519558676, + "loss": 2.9894, + "theoretical_loss": 3.888497069720885, + "tokens_seen": 532545536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042354062186559675, + "loss": 2.7836, + "theoretical_loss": 3.888446090660845, + "tokens_seen": 532611072 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423530591775326, + "loss": 2.7741, + "theoretical_loss": 3.88839511962936, + "tokens_seen": 532676608 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042352056168505517, + "loss": 3.0174, + "theoretical_loss": 3.8883441566241763, + "tokens_seen": 532742144 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042351053159478435, + "loss": 2.9035, + "theoretical_loss": 3.8882932016430436, + "tokens_seen": 532807680 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042350050150451353, + "loss": 2.735, + "theoretical_loss": 3.8882422546837123, + "tokens_seen": 532873216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042349047141424276, + "loss": 2.9197, + "theoretical_loss": 3.888191315743933, + "tokens_seen": 532938752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234804413239719, + "loss": 2.8019, + "theoretical_loss": 3.888140384821457, + "tokens_seen": 533004288 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042347041123370113, + "loss": 2.7563, + "theoretical_loss": 3.8880894619140367, + "tokens_seen": 533069824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042346038114343025, + "loss": 3.0713, + "theoretical_loss": 3.888038547019427, + "tokens_seen": 533135360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234503510531595, + "loss": 2.9346, + "theoretical_loss": 3.88798764013538, + "tokens_seen": 533200896 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042344032096288867, + "loss": 2.9696, + "theoretical_loss": 3.8879367412596535, + "tokens_seen": 533266432 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042343029087261785, + "loss": 2.8591, + "theoretical_loss": 3.887885850390002, + "tokens_seen": 533331968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234202607823471, + "loss": 2.7303, + "theoretical_loss": 3.8878349675241832, + "tokens_seen": 533397504 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004234102306920762, + "loss": 2.936, + "theoretical_loss": 3.8877840926599556, + "tokens_seen": 533463040 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042340020060180545, + "loss": 2.6803, + "theoretical_loss": 3.8877332257950776, + "tokens_seen": 533528576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042339017051153463, + "loss": 2.8323, + "theoretical_loss": 3.887682366927309, + "tokens_seen": 533594112 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233801404212638, + "loss": 2.9108, + "theoretical_loss": 3.8876315160544106, + "tokens_seen": 533659648 + }, + { + "epoch": 0.16, + "learning_rate": 0.000423370110330993, + "loss": 2.9117, + "theoretical_loss": 3.8875806731741447, + "tokens_seen": 533725184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233600802407222, + "loss": 2.8684, + "theoretical_loss": 3.887529838284273, + "tokens_seen": 533790720 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042335005015045135, + "loss": 2.7934, + "theoretical_loss": 3.887479011382559, + "tokens_seen": 533856256 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233400200601806, + "loss": 2.6989, + "theoretical_loss": 3.8874281924667677, + "tokens_seen": 533921792 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004233299899699097, + "loss": 2.6725, + "theoretical_loss": 3.887377381534664, + "tokens_seen": 533987328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042331995987963895, + "loss": 2.764, + "theoretical_loss": 3.8873265785840134, + "tokens_seen": 534052864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 304686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.990126848220825, + "objective/train/theoretical_loss": 3.8872757836125844, + "objective/train/tokens_used": 554578400, + "theoretical_loss": 3.8872757836125844, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042330992978936813, + "loss": 2.8816, + "theoretical_loss": 3.8872757836125844, + "tokens_seen": 534118400 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232998996990973, + "loss": 2.7513, + "theoretical_loss": 3.8872249966181442, + "tokens_seen": 534183936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232898696088265, + "loss": 2.9891, + "theoretical_loss": 3.887174217598461, + "tokens_seen": 534249472 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232798395185557, + "loss": 2.7367, + "theoretical_loss": 3.8871234465513056, + "tokens_seen": 534315008 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042326980942828486, + "loss": 3.0058, + "theoretical_loss": 3.8870726834744485, + "tokens_seen": 534380544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232597793380141, + "loss": 2.7603, + "theoretical_loss": 3.8870219283656597, + "tokens_seen": 534446080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232497492477432, + "loss": 2.8634, + "theoretical_loss": 3.886971181222714, + "tokens_seen": 534511616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042323971915747246, + "loss": 2.8499, + "theoretical_loss": 3.886920442043383, + "tokens_seen": 534577152 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232296890672016, + "loss": 2.9109, + "theoretical_loss": 3.886869710825442, + "tokens_seen": 534642688 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004232196589769308, + "loss": 2.884, + "theoretical_loss": 3.886818987566665, + "tokens_seen": 534708224 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042320962888666, + "loss": 2.5804, + "theoretical_loss": 3.8867682722648293, + "tokens_seen": 534773760 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231995987963892, + "loss": 2.7504, + "theoretical_loss": 3.886717564917711, + "tokens_seen": 534839296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042318956870611836, + "loss": 2.9655, + "theoretical_loss": 3.886666865523088, + "tokens_seen": 534904832 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231795386158476, + "loss": 2.7419, + "theoretical_loss": 3.886616174078739, + "tokens_seen": 534970368 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231695085255767, + "loss": 2.7549, + "theoretical_loss": 3.886565490582444, + "tokens_seen": 535035904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042315947843530596, + "loss": 2.9635, + "theoretical_loss": 3.8865148150319824, + "tokens_seen": 535101440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231494483450351, + "loss": 3.0665, + "theoretical_loss": 3.886464147425137, + "tokens_seen": 535166976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231394182547643, + "loss": 2.8279, + "theoretical_loss": 3.8864134877596896, + "tokens_seen": 535232512 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231293881644935, + "loss": 2.7782, + "theoretical_loss": 3.886362836033423, + "tokens_seen": 535298048 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004231193580742227, + "loss": 2.3214, + "theoretical_loss": 3.8863121922441213, + "tokens_seen": 535363584 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042310932798395186, + "loss": 2.8828, + "theoretical_loss": 3.8862615563895693, + "tokens_seen": 535429120 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042309929789368105, + "loss": 2.8513, + "theoretical_loss": 3.886210928467553, + "tokens_seen": 535494656 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230892678034102, + "loss": 2.9486, + "theoretical_loss": 3.8861603084758594, + "tokens_seen": 535560192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042307923771313946, + "loss": 2.845, + "theoretical_loss": 3.8861096964122757, + "tokens_seen": 535625728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230692076228686, + "loss": 2.7738, + "theoretical_loss": 3.886059092274591, + "tokens_seen": 535691264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 305312, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0122928619384766, + "objective/train/theoretical_loss": 3.886008496060594, + "objective/train/tokens_used": 556216800, + "theoretical_loss": 3.886008496060594, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230591775325978, + "loss": 2.9321, + "theoretical_loss": 3.886008496060594, + "tokens_seen": 535756800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042304914744232695, + "loss": 2.8908, + "theoretical_loss": 3.8859579077680753, + "tokens_seen": 535822336 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004230391173520562, + "loss": 2.6939, + "theoretical_loss": 3.8859073273948255, + "tokens_seen": 535887872 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042302908726178537, + "loss": 2.8025, + "theoretical_loss": 3.8858567549386374, + "tokens_seen": 535953408 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042301905717151455, + "loss": 2.9458, + "theoretical_loss": 3.885806190397304, + "tokens_seen": 536018944 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042300902708124373, + "loss": 2.6714, + "theoretical_loss": 3.8857556337686177, + "tokens_seen": 536084480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042299899699097297, + "loss": 2.8947, + "theoretical_loss": 3.8857050850503754, + "tokens_seen": 536150016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229889669007021, + "loss": 2.9313, + "theoretical_loss": 3.8856545442403707, + "tokens_seen": 536215552 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042297893681043133, + "loss": 2.8646, + "theoretical_loss": 3.885604011336401, + "tokens_seen": 536281088 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042296890672016045, + "loss": 2.829, + "theoretical_loss": 3.8855534863362635, + "tokens_seen": 536346624 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229588766298897, + "loss": 2.9151, + "theoretical_loss": 3.8855029692377565, + "tokens_seen": 536412160 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042294884653961887, + "loss": 3.0059, + "theoretical_loss": 3.885452460038679, + "tokens_seen": 536477696 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042293881644934805, + "loss": 2.7475, + "theoretical_loss": 3.8854019587368307, + "tokens_seen": 536543232 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042292878635907723, + "loss": 2.8329, + "theoretical_loss": 3.8853514653300136, + "tokens_seen": 536608768 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229187562688064, + "loss": 2.7283, + "theoretical_loss": 3.8853009798160274, + "tokens_seen": 536674304 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004229087261785356, + "loss": 2.659, + "theoretical_loss": 3.885250502192677, + "tokens_seen": 536739840 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042289869608826483, + "loss": 2.8879, + "theoretical_loss": 3.8852000324577647, + "tokens_seen": 536805376 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042288866599799396, + "loss": 2.8903, + "theoretical_loss": 3.8851495706090944, + "tokens_seen": 536870912 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228786359077232, + "loss": 2.5484, + "theoretical_loss": 3.8850991166444726, + "tokens_seen": 536936448 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228686058174523, + "loss": 2.8039, + "theoretical_loss": 3.885048670561705, + "tokens_seen": 537001984 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042285857572718155, + "loss": 2.8749, + "theoretical_loss": 3.8849982323585985, + "tokens_seen": 537067520 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042284854563691074, + "loss": 2.8962, + "theoretical_loss": 3.8849478020329604, + "tokens_seen": 537133056 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228385155466399, + "loss": 2.9466, + "theoretical_loss": 3.8848973795826005, + "tokens_seen": 537198592 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004228284854563691, + "loss": 2.796, + "theoretical_loss": 3.884846965005328, + "tokens_seen": 537264128 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042281845536609833, + "loss": 2.6911, + "theoretical_loss": 3.8847965582989534, + "tokens_seen": 537329664 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 306391, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.048434257507324, + "objective/train/theoretical_loss": 3.8847461594612884, + "objective/train/tokens_used": 557855200, + "theoretical_loss": 3.8847461594612884, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042280842527582746, + "loss": 2.8548, + "theoretical_loss": 3.8847461594612884, + "tokens_seen": 537395200 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227983951855567, + "loss": 2.8251, + "theoretical_loss": 3.884695768490145, + "tokens_seen": 537460736 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227883650952858, + "loss": 2.8225, + "theoretical_loss": 3.8846453853833367, + "tokens_seen": 537526272 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042277833500501506, + "loss": 2.9452, + "theoretical_loss": 3.8845950101386766, + "tokens_seen": 537591808 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042276830491474424, + "loss": 2.7989, + "theoretical_loss": 3.8845446427539807, + "tokens_seen": 537657344 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227582748244734, + "loss": 2.6692, + "theoretical_loss": 3.884494283227064, + "tokens_seen": 537722880 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227482447342026, + "loss": 2.9536, + "theoretical_loss": 3.884443931555744, + "tokens_seen": 537788416 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227382146439318, + "loss": 3.0258, + "theoretical_loss": 3.884393587737838, + "tokens_seen": 537853952 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042272818455366096, + "loss": 2.7838, + "theoretical_loss": 3.8843432517711634, + "tokens_seen": 537919488 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227181544633902, + "loss": 2.7952, + "theoretical_loss": 3.8842929236535406, + "tokens_seen": 537985024 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004227081243731193, + "loss": 3.0073, + "theoretical_loss": 3.8842426033827895, + "tokens_seen": 538050560 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042269809428284856, + "loss": 2.7916, + "theoretical_loss": 3.8841922909567312, + "tokens_seen": 538116096 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226880641925777, + "loss": 2.8229, + "theoretical_loss": 3.8841419863731863, + "tokens_seen": 538181632 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226780341023069, + "loss": 2.6835, + "theoretical_loss": 3.8840916896299795, + "tokens_seen": 538247168 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042266800401203616, + "loss": 2.8953, + "theoretical_loss": 3.8840414007249335, + "tokens_seen": 538312704 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226579739217653, + "loss": 2.7475, + "theoretical_loss": 3.8839911196558727, + "tokens_seen": 538378240 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226479438314945, + "loss": 2.9781, + "theoretical_loss": 3.8839408464206224, + "tokens_seen": 538443776 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226379137412237, + "loss": 3.0243, + "theoretical_loss": 3.8838905810170097, + "tokens_seen": 538509312 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004226278836509529, + "loss": 3.0065, + "theoretical_loss": 3.8838403234428602, + "tokens_seen": 538574848 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042261785356068206, + "loss": 2.9446, + "theoretical_loss": 3.8837900736960034, + "tokens_seen": 538640384 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042260782347041125, + "loss": 2.8828, + "theoretical_loss": 3.8837398317742666, + "tokens_seen": 538705920 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225977933801404, + "loss": 2.9098, + "theoretical_loss": 3.883689597675481, + "tokens_seen": 538771456 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042258776328986966, + "loss": 2.9336, + "theoretical_loss": 3.8836393713974764, + "tokens_seen": 538836992 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225777331995988, + "loss": 3.1058, + "theoretical_loss": 3.883589152938084, + "tokens_seen": 538902528 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422567703109328, + "loss": 2.8419, + "theoretical_loss": 3.8835389422951367, + "tokens_seen": 538968064 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 307598, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.633847236633301, + "objective/train/theoretical_loss": 3.883488739466467, + "objective/train/tokens_used": 559493600, + "theoretical_loss": 3.883488739466467, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042255767301905715, + "loss": 2.788, + "theoretical_loss": 3.883488739466467, + "tokens_seen": 539033600 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004225476429287864, + "loss": 2.76, + "theoretical_loss": 3.8834385444499095, + "tokens_seen": 539099136 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042253761283851557, + "loss": 2.8685, + "theoretical_loss": 3.883388357243299, + "tokens_seen": 539164672 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042252758274824475, + "loss": 2.7135, + "theoretical_loss": 3.883338177844471, + "tokens_seen": 539230208 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042251755265797393, + "loss": 2.9524, + "theoretical_loss": 3.883288006251262, + "tokens_seen": 539295744 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042250752256770317, + "loss": 2.871, + "theoretical_loss": 3.88323784246151, + "tokens_seen": 539361280 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224974924774323, + "loss": 2.7594, + "theoretical_loss": 3.883187686473053, + "tokens_seen": 539426816 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042248746238716153, + "loss": 2.8279, + "theoretical_loss": 3.8831375382837305, + "tokens_seen": 539492352 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042247743229689065, + "loss": 2.7441, + "theoretical_loss": 3.883087397891382, + "tokens_seen": 539557888 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224674022066199, + "loss": 3.0017, + "theoretical_loss": 3.883037265293849, + "tokens_seen": 539623424 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042245737211634907, + "loss": 2.7708, + "theoretical_loss": 3.8829871404889724, + "tokens_seen": 539688960 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042244734202607825, + "loss": 2.9023, + "theoretical_loss": 3.8829370234745957, + "tokens_seen": 539754496 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042243731193580743, + "loss": 2.7448, + "theoretical_loss": 3.8828869142485622, + "tokens_seen": 539820032 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224272818455366, + "loss": 3.064, + "theoretical_loss": 3.8828368128087165, + "tokens_seen": 539885568 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004224172517552658, + "loss": 2.834, + "theoretical_loss": 3.882786719152903, + "tokens_seen": 539951104 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042240722166499503, + "loss": 2.7914, + "theoretical_loss": 3.8827366332789683, + "tokens_seen": 540016640 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042239719157472416, + "loss": 2.7571, + "theoretical_loss": 3.882686555184759, + "tokens_seen": 540082176 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223871614844534, + "loss": 2.812, + "theoretical_loss": 3.882636484868124, + "tokens_seen": 540147712 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223771313941825, + "loss": 3.0633, + "theoretical_loss": 3.882586422326911, + "tokens_seen": 540213248 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042236710130391176, + "loss": 2.8002, + "theoretical_loss": 3.882536367558969, + "tokens_seen": 540278784 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042235707121364094, + "loss": 2.7051, + "theoretical_loss": 3.8824863205621494, + "tokens_seen": 540344320 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223470411233701, + "loss": 2.7578, + "theoretical_loss": 3.882436281334303, + "tokens_seen": 540409856 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223370110330993, + "loss": 2.8951, + "theoretical_loss": 3.8823862498732815, + "tokens_seen": 540475392 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042232698094282853, + "loss": 2.7626, + "theoretical_loss": 3.882336226176939, + "tokens_seen": 540540928 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042231695085255766, + "loss": 2.8586, + "theoretical_loss": 3.8822862102431275, + "tokens_seen": 540606464 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 308194, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8666329383850098, + "objective/train/theoretical_loss": 3.882236202069703, + "objective/train/tokens_used": 561132000, + "theoretical_loss": 3.882236202069703, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004223069207622869, + "loss": 2.8035, + "theoretical_loss": 3.882236202069703, + "tokens_seen": 540672000 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422296890672016, + "loss": 2.8564, + "theoretical_loss": 3.8821862016545206, + "tokens_seen": 540737536 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042228686058174526, + "loss": 2.959, + "theoretical_loss": 3.8821362089954365, + "tokens_seen": 540803072 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042227683049147444, + "loss": 2.827, + "theoretical_loss": 3.8820862240903082, + "tokens_seen": 540868608 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222668004012036, + "loss": 2.8683, + "theoretical_loss": 3.882036246936993, + "tokens_seen": 540934144 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222567703109328, + "loss": 2.9829, + "theoretical_loss": 3.8819862775333505, + "tokens_seen": 540999680 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422246740220662, + "loss": 2.9701, + "theoretical_loss": 3.88193631587724, + "tokens_seen": 541065216 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042223671013039116, + "loss": 2.8898, + "theoretical_loss": 3.8818863619665223, + "tokens_seen": 541130752 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222266800401204, + "loss": 2.7672, + "theoretical_loss": 3.881836415799059, + "tokens_seen": 541196288 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004222166499498495, + "loss": 2.9484, + "theoretical_loss": 3.881786477372712, + "tokens_seen": 541261824 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042220661985957876, + "loss": 2.8886, + "theoretical_loss": 3.881736546685345, + "tokens_seen": 541327360 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221965897693079, + "loss": 3.0252, + "theoretical_loss": 3.8816866237348218, + "tokens_seen": 541392896 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221865596790371, + "loss": 2.8781, + "theoretical_loss": 3.8816367085190064, + "tokens_seen": 541458432 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221765295887663, + "loss": 2.9667, + "theoretical_loss": 3.8815868010357657, + "tokens_seen": 541523968 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221664994984955, + "loss": 2.7153, + "theoretical_loss": 3.8815369012829657, + "tokens_seen": 541589504 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042215646940822467, + "loss": 2.5785, + "theoretical_loss": 3.881487009258473, + "tokens_seen": 541655040 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221464393179539, + "loss": 2.8231, + "theoretical_loss": 3.881437124960157, + "tokens_seen": 541720576 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042213640922768303, + "loss": 2.7915, + "theoretical_loss": 3.8813872483858867, + "tokens_seen": 541786112 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042212637913741226, + "loss": 2.8312, + "theoretical_loss": 3.881337379533531, + "tokens_seen": 541851648 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221163490471414, + "loss": 2.7949, + "theoretical_loss": 3.8812875184009616, + "tokens_seen": 541917184 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004221063189568706, + "loss": 2.8074, + "theoretical_loss": 3.88123766498605, + "tokens_seen": 541982720 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220962888665998, + "loss": 2.7497, + "theoretical_loss": 3.881187819286668, + "tokens_seen": 542048256 + }, + { + "epoch": 0.16, + "learning_rate": 0.000422086258776329, + "loss": 2.9072, + "theoretical_loss": 3.8811379813006894, + "tokens_seen": 542113792 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042207622868605817, + "loss": 2.9012, + "theoretical_loss": 3.8810881510259883, + "tokens_seen": 542179328 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042206619859578735, + "loss": 2.6692, + "theoretical_loss": 3.8810383284604395, + "tokens_seen": 542244864 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 309537, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.619420289993286, + "objective/train/theoretical_loss": 3.8809885136019195, + "objective/train/tokens_used": 562770400, + "theoretical_loss": 3.8809885136019195, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042205616850551653, + "loss": 2.9674, + "theoretical_loss": 3.8809885136019195, + "tokens_seen": 542310400 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042204613841524577, + "loss": 2.7469, + "theoretical_loss": 3.880938706448304, + "tokens_seen": 542375936 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220361083249749, + "loss": 2.8841, + "theoretical_loss": 3.8808889069974706, + "tokens_seen": 542441472 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042202607823470413, + "loss": 2.8074, + "theoretical_loss": 3.880839115247298, + "tokens_seen": 542507008 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220160481444333, + "loss": 2.7575, + "theoretical_loss": 3.880789331195665, + "tokens_seen": 542572544 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004220060180541625, + "loss": 2.8394, + "theoretical_loss": 3.8807395548404524, + "tokens_seen": 542638080 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004219959879638917, + "loss": 2.7962, + "theoretical_loss": 3.8806897861795404, + "tokens_seen": 542703616 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042198595787362085, + "loss": 2.9498, + "theoretical_loss": 3.880640025210811, + "tokens_seen": 542769152 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042197592778335004, + "loss": 2.7269, + "theoretical_loss": 3.880590271932147, + "tokens_seen": 542834688 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042196589769307927, + "loss": 2.8028, + "theoretical_loss": 3.8805405263414308, + "tokens_seen": 542900224 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004219558676028084, + "loss": 2.8841, + "theoretical_loss": 3.8804907884365476, + "tokens_seen": 542965760 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042194583751253763, + "loss": 3.0371, + "theoretical_loss": 3.8804410582153817, + "tokens_seen": 543031296 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042193580742226676, + "loss": 2.912, + "theoretical_loss": 3.8803913356758195, + "tokens_seen": 543096832 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421925777331996, + "loss": 2.7219, + "theoretical_loss": 3.8803416208157477, + "tokens_seen": 543162368 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042191574724172523, + "loss": 2.6821, + "theoretical_loss": 3.8802919136330534, + "tokens_seen": 543227904 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042190571715145436, + "loss": 2.8521, + "theoretical_loss": 3.8802422141256256, + "tokens_seen": 543293440 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218956870611836, + "loss": 2.7739, + "theoretical_loss": 3.8801925222913534, + "tokens_seen": 543358976 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218856569709127, + "loss": 2.9423, + "theoretical_loss": 3.880142838128126, + "tokens_seen": 543424512 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042187562688064196, + "loss": 2.7782, + "theoretical_loss": 3.880093161633836, + "tokens_seen": 543490048 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042186559679037114, + "loss": 2.7181, + "theoretical_loss": 3.8800434928063736, + "tokens_seen": 543555584 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218555667001003, + "loss": 2.8917, + "theoretical_loss": 3.879993831643632, + "tokens_seen": 543621120 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218455366098295, + "loss": 3.0724, + "theoretical_loss": 3.879944178143505, + "tokens_seen": 543686656 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042183550651955873, + "loss": 2.8608, + "theoretical_loss": 3.879894532303886, + "tokens_seen": 543752192 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042182547642928786, + "loss": 2.8151, + "theoretical_loss": 3.8798448941226704, + "tokens_seen": 543817728 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218154463390171, + "loss": 2.7074, + "theoretical_loss": 3.879795263597754, + "tokens_seen": 543883264 + }, + { + "epoch": 0.16, + "objective/train/docs_used": 310176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.820920705795288, + "objective/train/theoretical_loss": 3.8797456407270343, + "objective/train/tokens_used": 564408800, + "theoretical_loss": 3.8797456407270343, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004218054162487462, + "loss": 2.8936, + "theoretical_loss": 3.8797456407270343, + "tokens_seen": 543948800 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042179538615847546, + "loss": 2.8416, + "theoretical_loss": 3.879696025508408, + "tokens_seen": 544014336 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042178535606820464, + "loss": 2.9673, + "theoretical_loss": 3.8796464179397736, + "tokens_seen": 544079872 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217753259779338, + "loss": 3.0778, + "theoretical_loss": 3.8795968180190306, + "tokens_seen": 544145408 + }, + { + "epoch": 0.16, + "learning_rate": 0.000421765295887663, + "loss": 2.9471, + "theoretical_loss": 3.8795472257440786, + "tokens_seen": 544210944 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217552657973922, + "loss": 2.7651, + "theoretical_loss": 3.8794976411128186, + "tokens_seen": 544276480 + }, + { + "epoch": 0.16, + "learning_rate": 0.00042174523570712136, + "loss": 2.761, + "theoretical_loss": 3.8794480641231535, + "tokens_seen": 544342016 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217352056168506, + "loss": 3.0898, + "theoretical_loss": 3.879398494772984, + "tokens_seen": 544407552 + }, + { + "epoch": 0.16, + "learning_rate": 0.0004217251755265797, + "loss": 2.7585, + "theoretical_loss": 3.879348933060215, + "tokens_seen": 544473088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042171514543630896, + "loss": 2.9004, + "theoretical_loss": 3.8792993789827497, + "tokens_seen": 544538624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004217051153460381, + "loss": 2.7262, + "theoretical_loss": 3.879249832538493, + "tokens_seen": 544604160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216950852557673, + "loss": 2.8847, + "theoretical_loss": 3.879200293725352, + "tokens_seen": 544669696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216850551654965, + "loss": 2.6465, + "theoretical_loss": 3.879150762541232, + "tokens_seen": 544735232 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216750250752257, + "loss": 2.953, + "theoretical_loss": 3.8791012389840414, + "tokens_seen": 544800768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042166499498495487, + "loss": 2.6372, + "theoretical_loss": 3.879051723051688, + "tokens_seen": 544866304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216549648946841, + "loss": 2.8553, + "theoretical_loss": 3.8790022147420817, + "tokens_seen": 544931840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042164493480441323, + "loss": 2.861, + "theoretical_loss": 3.8789527140531312, + "tokens_seen": 544997376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042163490471414246, + "loss": 2.9117, + "theoretical_loss": 3.8789032209827488, + "tokens_seen": 545062912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216248746238716, + "loss": 2.7233, + "theoretical_loss": 3.8788537355288444, + "tokens_seen": 545128448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004216148445336008, + "loss": 2.8419, + "theoretical_loss": 3.8788042576893322, + "tokens_seen": 545193984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042160481444333, + "loss": 2.7416, + "theoretical_loss": 3.8787547874621247, + "tokens_seen": 545259520 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215947843530592, + "loss": 2.9282, + "theoretical_loss": 3.8787053248451357, + "tokens_seen": 545325056 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042158475426278837, + "loss": 2.8425, + "theoretical_loss": 3.8786558698362805, + "tokens_seen": 545390592 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042157472417251755, + "loss": 2.9664, + "theoretical_loss": 3.8786064224334744, + "tokens_seen": 545456128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042156469408224673, + "loss": 2.6686, + "theoretical_loss": 3.8785569826346347, + "tokens_seen": 545521664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 311361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7344307899475098, + "objective/train/theoretical_loss": 3.8785075504376785, + "objective/train/tokens_used": 566047200, + "theoretical_loss": 3.8785075504376785, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042155466399197597, + "loss": 2.9008, + "theoretical_loss": 3.8785075504376785, + "tokens_seen": 545587200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215446339017051, + "loss": 2.8395, + "theoretical_loss": 3.878458125840524, + "tokens_seen": 545652736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042153460381143433, + "loss": 2.852, + "theoretical_loss": 3.87840870884109, + "tokens_seen": 545718272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215245737211635, + "loss": 2.828, + "theoretical_loss": 3.878359299437296, + "tokens_seen": 545783808 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215145436308927, + "loss": 2.9685, + "theoretical_loss": 3.878309897627064, + "tokens_seen": 545849344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004215045135406219, + "loss": 2.9763, + "theoretical_loss": 3.878260503408314, + "tokens_seen": 545914880 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042149448345035105, + "loss": 2.9497, + "theoretical_loss": 3.878211116778969, + "tokens_seen": 545980416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042148445336008024, + "loss": 2.9101, + "theoretical_loss": 3.878161737736952, + "tokens_seen": 546045952 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042147442326980947, + "loss": 2.5154, + "theoretical_loss": 3.8781123662801873, + "tokens_seen": 546111488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214643931795386, + "loss": 2.8239, + "theoretical_loss": 3.878063002406599, + "tokens_seen": 546177024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042145436308926783, + "loss": 2.9453, + "theoretical_loss": 3.878013646114113, + "tokens_seen": 546242560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042144433299899696, + "loss": 2.692, + "theoretical_loss": 3.8779642974006556, + "tokens_seen": 546308096 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214343029087262, + "loss": 2.7597, + "theoretical_loss": 3.877914956264154, + "tokens_seen": 546373632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004214242728184554, + "loss": 2.8578, + "theoretical_loss": 3.8778656227025365, + "tokens_seen": 546439168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042141424272818456, + "loss": 2.9577, + "theoretical_loss": 3.877816296713732, + "tokens_seen": 546504704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042140421263791374, + "loss": 2.9289, + "theoretical_loss": 3.8777669782956696, + "tokens_seen": 546570240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213941825476429, + "loss": 2.9022, + "theoretical_loss": 3.8777176674462797, + "tokens_seen": 546635776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213841524573721, + "loss": 2.8726, + "theoretical_loss": 3.877668364163494, + "tokens_seen": 546701312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042137412236710134, + "loss": 2.6827, + "theoretical_loss": 3.8776190684452447, + "tokens_seen": 546766848 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042136409227683046, + "loss": 2.8975, + "theoretical_loss": 3.8775697802894644, + "tokens_seen": 546832384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213540621865597, + "loss": 2.8235, + "theoretical_loss": 3.8775204996940866, + "tokens_seen": 546897920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213440320962889, + "loss": 2.7518, + "theoretical_loss": 3.877471226657047, + "tokens_seen": 546963456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042133400200601806, + "loss": 2.9066, + "theoretical_loss": 3.877421961176279, + "tokens_seen": 547028992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042132397191574724, + "loss": 3.0313, + "theoretical_loss": 3.87737270324972, + "tokens_seen": 547094528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213139418254764, + "loss": 3.0107, + "theoretical_loss": 3.8773234528753076, + "tokens_seen": 547160064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 312508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.020892381668091, + "objective/train/theoretical_loss": 3.8772742100509774, + "objective/train/tokens_used": 567685600, + "theoretical_loss": 3.8772742100509774, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004213039117352056, + "loss": 2.8083, + "theoretical_loss": 3.8772742100509774, + "tokens_seen": 547225600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042129388164493484, + "loss": 2.5294, + "theoretical_loss": 3.8772249747746703, + "tokens_seen": 547291136 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042128385155466397, + "loss": 3.0742, + "theoretical_loss": 3.8771757470443244, + "tokens_seen": 547356672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212738214643932, + "loss": 2.6558, + "theoretical_loss": 3.87712652685788, + "tokens_seen": 547422208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042126379137412233, + "loss": 3.0363, + "theoretical_loss": 3.8770773142132784, + "tokens_seen": 547487744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042125376128385156, + "loss": 2.8451, + "theoretical_loss": 3.8770281091084615, + "tokens_seen": 547553280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042124373119358075, + "loss": 2.7166, + "theoretical_loss": 3.8769789115413715, + "tokens_seen": 547618816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212337011033099, + "loss": 2.7842, + "theoretical_loss": 3.8769297215099523, + "tokens_seen": 547684352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212236710130391, + "loss": 3.1148, + "theoretical_loss": 3.8768805390121477, + "tokens_seen": 547749888 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004212136409227683, + "loss": 2.8194, + "theoretical_loss": 3.876831364045903, + "tokens_seen": 547815424 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042120361083249747, + "loss": 2.7916, + "theoretical_loss": 3.876782196609164, + "tokens_seen": 547880960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211935807422267, + "loss": 2.4156, + "theoretical_loss": 3.8767330366998776, + "tokens_seen": 547946496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042118355065195583, + "loss": 2.9464, + "theoretical_loss": 3.8766838843159905, + "tokens_seen": 548012032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042117352056168507, + "loss": 2.8205, + "theoretical_loss": 3.8766347394554517, + "tokens_seen": 548077568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211634904714143, + "loss": 2.9376, + "theoretical_loss": 3.87658560211621, + "tokens_seen": 548143104 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042115346038114343, + "loss": 2.8582, + "theoretical_loss": 3.8765364722962152, + "tokens_seen": 548208640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042114343029087266, + "loss": 2.9846, + "theoretical_loss": 3.8764873499934183, + "tokens_seen": 548274176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211334002006018, + "loss": 2.7357, + "theoretical_loss": 3.8764382352057707, + "tokens_seen": 548339712 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042112337011033103, + "loss": 2.911, + "theoretical_loss": 3.8763891279312244, + "tokens_seen": 548405248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211133400200602, + "loss": 2.8353, + "theoretical_loss": 3.876340028167732, + "tokens_seen": 548470784 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004211033099297894, + "loss": 2.8009, + "theoretical_loss": 3.876290935913249, + "tokens_seen": 548536320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042109327983951857, + "loss": 2.7628, + "theoretical_loss": 3.876241851165729, + "tokens_seen": 548601856 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042108324974924775, + "loss": 2.74, + "theoretical_loss": 3.876192773923127, + "tokens_seen": 548667392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042107321965897693, + "loss": 3.0434, + "theoretical_loss": 3.8761437041834004, + "tokens_seen": 548732928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042106318956870617, + "loss": 2.8916, + "theoretical_loss": 3.8760946419445057, + "tokens_seen": 548798464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 312943, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1393747329711914, + "objective/train/theoretical_loss": 3.876045587204401, + "objective/train/tokens_used": 569324000, + "theoretical_loss": 3.876045587204401, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210531594784353, + "loss": 2.7268, + "theoretical_loss": 3.876045587204401, + "tokens_seen": 548864000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042104312938816453, + "loss": 2.7706, + "theoretical_loss": 3.875996539961045, + "tokens_seen": 548929536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210330992978937, + "loss": 2.8024, + "theoretical_loss": 3.875947500212397, + "tokens_seen": 548995072 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210230692076229, + "loss": 2.8611, + "theoretical_loss": 3.8758984679564175, + "tokens_seen": 549060608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004210130391173521, + "loss": 2.5877, + "theoretical_loss": 3.875849443191067, + "tokens_seen": 549126144 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042100300902708125, + "loss": 2.7126, + "theoretical_loss": 3.875800425914308, + "tokens_seen": 549191680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042099297893681044, + "loss": 2.8382, + "theoretical_loss": 3.8757514161241033, + "tokens_seen": 549257216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042098294884653967, + "loss": 2.8309, + "theoretical_loss": 3.8757024138184164, + "tokens_seen": 549322752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209729187562688, + "loss": 2.7394, + "theoretical_loss": 3.8756534189952108, + "tokens_seen": 549388288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042096288866599803, + "loss": 2.8002, + "theoretical_loss": 3.8756044316524525, + "tokens_seen": 549453824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042095285857572716, + "loss": 2.8171, + "theoretical_loss": 3.875555451788107, + "tokens_seen": 549519360 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209428284854564, + "loss": 2.8046, + "theoretical_loss": 3.8755064794001406, + "tokens_seen": 549584896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209327983951856, + "loss": 2.8639, + "theoretical_loss": 3.875457514486521, + "tokens_seen": 549650432 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042092276830491476, + "loss": 2.8338, + "theoretical_loss": 3.8754085570452164, + "tokens_seen": 549715968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042091273821464394, + "loss": 2.9081, + "theoretical_loss": 3.8753596070741967, + "tokens_seen": 549781504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004209027081243731, + "loss": 2.5443, + "theoretical_loss": 3.8753106645714306, + "tokens_seen": 549847040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208926780341023, + "loss": 2.587, + "theoretical_loss": 3.8752617295348895, + "tokens_seen": 549912576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042088264794383154, + "loss": 2.7818, + "theoretical_loss": 3.8752128019625443, + "tokens_seen": 549978112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042087261785356066, + "loss": 2.8385, + "theoretical_loss": 3.8751638818523677, + "tokens_seen": 550043648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208625877632899, + "loss": 2.7451, + "theoretical_loss": 3.875114969202332, + "tokens_seen": 550109184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208525576730191, + "loss": 2.7683, + "theoretical_loss": 3.8750660640104124, + "tokens_seen": 550174720 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042084252758274826, + "loss": 2.8129, + "theoretical_loss": 3.875017166274582, + "tokens_seen": 550240256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042083249749247744, + "loss": 3.0056, + "theoretical_loss": 3.8749682759928166, + "tokens_seen": 550305792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208224674022066, + "loss": 2.8207, + "theoretical_loss": 3.8749193931630925, + "tokens_seen": 550371328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004208124373119358, + "loss": 2.9711, + "theoretical_loss": 3.8748705177833873, + "tokens_seen": 550436864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 314102, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.005920886993408, + "objective/train/theoretical_loss": 3.874821649851678, + "objective/train/tokens_used": 570962400, + "theoretical_loss": 3.874821649851678, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042080240722166504, + "loss": 3.0135, + "theoretical_loss": 3.874821649851678, + "tokens_seen": 550502400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042079237713139417, + "loss": 2.6148, + "theoretical_loss": 3.8747727893659434, + "tokens_seen": 550567936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207823470411234, + "loss": 2.7024, + "theoretical_loss": 3.8747239363241626, + "tokens_seen": 550633472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042077231695085253, + "loss": 2.698, + "theoretical_loss": 3.8746750907243164, + "tokens_seen": 550699008 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042076228686058176, + "loss": 3.0277, + "theoretical_loss": 3.8746262525643846, + "tokens_seen": 550764544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042075225677031095, + "loss": 2.8225, + "theoretical_loss": 3.8745774218423508, + "tokens_seen": 550830080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207422266800401, + "loss": 2.9363, + "theoretical_loss": 3.874528598556195, + "tokens_seen": 550895616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207321965897693, + "loss": 2.7584, + "theoretical_loss": 3.8744797827039026, + "tokens_seen": 550961152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207221664994985, + "loss": 2.7688, + "theoretical_loss": 3.874430974283456, + "tokens_seen": 551026688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042071213640922767, + "loss": 2.9367, + "theoretical_loss": 3.874382173292842, + "tokens_seen": 551092224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004207021063189569, + "loss": 2.7892, + "theoretical_loss": 3.874333379730044, + "tokens_seen": 551157760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042069207622868603, + "loss": 2.8285, + "theoretical_loss": 3.8742845935930506, + "tokens_seen": 551223296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042068204613841527, + "loss": 2.6297, + "theoretical_loss": 3.8742358148798473, + "tokens_seen": 551288832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042067201604814445, + "loss": 2.8855, + "theoretical_loss": 3.8741870435884227, + "tokens_seen": 551354368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042066198595787363, + "loss": 2.7368, + "theoretical_loss": 3.874138279716766, + "tokens_seen": 551419904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206519558676028, + "loss": 2.7036, + "theoretical_loss": 3.8740895232628665, + "tokens_seen": 551485440 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420641925777332, + "loss": 2.9885, + "theoretical_loss": 3.874040774224714, + "tokens_seen": 551550976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206318956870612, + "loss": 2.836, + "theoretical_loss": 3.8739920326003006, + "tokens_seen": 551616512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004206218655967904, + "loss": 2.8656, + "theoretical_loss": 3.8739432983876174, + "tokens_seen": 551682048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042061183550651954, + "loss": 2.766, + "theoretical_loss": 3.8738945715846578, + "tokens_seen": 551747584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042060180541624877, + "loss": 2.8527, + "theoretical_loss": 3.8738458521894144, + "tokens_seen": 551813120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205917753259779, + "loss": 2.8719, + "theoretical_loss": 3.8737971401998825, + "tokens_seen": 551878656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042058174523570713, + "loss": 2.7869, + "theoretical_loss": 3.8737484356140564, + "tokens_seen": 551944192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205717151454363, + "loss": 2.759, + "theoretical_loss": 3.873699738429932, + "tokens_seen": 552009728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205616850551655, + "loss": 2.8348, + "theoretical_loss": 3.873651048645506, + "tokens_seen": 552075264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 314596, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6391680240631104, + "objective/train/theoretical_loss": 3.873602366258776, + "objective/train/tokens_used": 572600800, + "theoretical_loss": 3.873602366258776, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205516549648947, + "loss": 2.7018, + "theoretical_loss": 3.873602366258776, + "tokens_seen": 552140800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205416248746239, + "loss": 3.0718, + "theoretical_loss": 3.87355369126774, + "tokens_seen": 552206336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042053159478435304, + "loss": 2.9741, + "theoretical_loss": 3.873505023670397, + "tokens_seen": 552271872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205215646940823, + "loss": 2.6878, + "theoretical_loss": 3.8734563634647463, + "tokens_seen": 552337408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004205115346038114, + "loss": 2.7524, + "theoretical_loss": 3.8734077106487894, + "tokens_seen": 552402944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042050150451354064, + "loss": 2.8489, + "theoretical_loss": 3.8733590652205265, + "tokens_seen": 552468480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204914744232698, + "loss": 2.8195, + "theoretical_loss": 3.8733104271779606, + "tokens_seen": 552534016 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420481444332999, + "loss": 2.7318, + "theoretical_loss": 3.8732617965190936, + "tokens_seen": 552599552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204714142427282, + "loss": 2.7373, + "theoretical_loss": 3.873213173241929, + "tokens_seen": 552665088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042046138415245736, + "loss": 2.7681, + "theoretical_loss": 3.8731645573444724, + "tokens_seen": 552730624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042045135406218654, + "loss": 2.801, + "theoretical_loss": 3.8731159488247284, + "tokens_seen": 552796160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204413239719158, + "loss": 2.7685, + "theoretical_loss": 3.8730673476807023, + "tokens_seen": 552861696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204312938816449, + "loss": 2.8712, + "theoretical_loss": 3.8730187539104017, + "tokens_seen": 552927232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042042126379137414, + "loss": 2.7277, + "theoretical_loss": 3.8729701675118338, + "tokens_seen": 552992768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204112337011033, + "loss": 2.8187, + "theoretical_loss": 3.8729215884830066, + "tokens_seen": 553058304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004204012036108325, + "loss": 2.6754, + "theoretical_loss": 3.8728730168219294, + "tokens_seen": 553123840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042039117352056174, + "loss": 2.6005, + "theoretical_loss": 3.872824452526612, + "tokens_seen": 553189376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042038114343029086, + "loss": 2.6217, + "theoretical_loss": 3.8727758955950646, + "tokens_seen": 553254912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203711133400201, + "loss": 2.9321, + "theoretical_loss": 3.8727273460252993, + "tokens_seen": 553320448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203610832497493, + "loss": 2.6787, + "theoretical_loss": 3.8726788038153277, + "tokens_seen": 553385984 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042035105315947846, + "loss": 2.8041, + "theoretical_loss": 3.872630268963163, + "tokens_seen": 553451520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042034102306920764, + "loss": 2.811, + "theoretical_loss": 3.872581741466818, + "tokens_seen": 553517056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004203309929789368, + "loss": 2.6364, + "theoretical_loss": 3.8725332213243084, + "tokens_seen": 553582592 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420320962888666, + "loss": 2.6177, + "theoretical_loss": 3.8724847085336487, + "tokens_seen": 553648128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042031093279839524, + "loss": 2.7531, + "theoretical_loss": 3.872436203092855, + "tokens_seen": 553713664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 315339, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5412378311157227, + "objective/train/theoretical_loss": 3.8723877049999444, + "objective/train/tokens_used": 574239200, + "theoretical_loss": 3.8723877049999444, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042030090270812437, + "loss": 2.4705, + "theoretical_loss": 3.8723877049999444, + "tokens_seen": 553779200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202908726178536, + "loss": 2.6838, + "theoretical_loss": 3.872339214252934, + "tokens_seen": 553844736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042028084252758273, + "loss": 2.6182, + "theoretical_loss": 3.8722907308498415, + "tokens_seen": 553910272 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042027081243731196, + "loss": 2.8419, + "theoretical_loss": 3.8722422547886874, + "tokens_seen": 553975808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042026078234704115, + "loss": 2.7933, + "theoretical_loss": 3.8721937860674904, + "tokens_seen": 554041344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202507522567703, + "loss": 2.9367, + "theoretical_loss": 3.872145324684272, + "tokens_seen": 554106880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202407221664995, + "loss": 2.5143, + "theoretical_loss": 3.8720968706370527, + "tokens_seen": 554172416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202306920762287, + "loss": 2.8308, + "theoretical_loss": 3.8720484239238555, + "tokens_seen": 554237952 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042022066198595787, + "loss": 2.707, + "theoretical_loss": 3.8719999845427022, + "tokens_seen": 554303488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004202106318956871, + "loss": 2.4634, + "theoretical_loss": 3.871951552491617, + "tokens_seen": 554369024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042020060180541623, + "loss": 2.6944, + "theoretical_loss": 3.871903127768625, + "tokens_seen": 554434560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042019057171514547, + "loss": 2.8056, + "theoretical_loss": 3.8718547103717507, + "tokens_seen": 554500096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042018054162487465, + "loss": 2.7418, + "theoretical_loss": 3.87180630029902, + "tokens_seen": 554565632 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042017051153460383, + "loss": 2.9632, + "theoretical_loss": 3.87175789754846, + "tokens_seen": 554631168 + }, + { + "epoch": 0.17, + "learning_rate": 0.000420160481444333, + "loss": 2.5859, + "theoretical_loss": 3.8717095021180983, + "tokens_seen": 554696704 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201504513540622, + "loss": 2.8359, + "theoretical_loss": 3.8716611140059625, + "tokens_seen": 554762240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201404212637914, + "loss": 2.6759, + "theoretical_loss": 3.871612733210082, + "tokens_seen": 554827776 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201303911735206, + "loss": 2.7389, + "theoretical_loss": 3.8715643597284863, + "tokens_seen": 554893312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042012036108324974, + "loss": 2.7195, + "theoretical_loss": 3.871515993559207, + "tokens_seen": 554958848 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042011033099297897, + "loss": 2.814, + "theoretical_loss": 3.871467634700274, + "tokens_seen": 555024384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004201003009027081, + "loss": 2.5207, + "theoretical_loss": 3.8714192831497205, + "tokens_seen": 555089920 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042009027081243733, + "loss": 2.6039, + "theoretical_loss": 3.871370938905579, + "tokens_seen": 555155456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200802407221665, + "loss": 3.1536, + "theoretical_loss": 3.8713226019658826, + "tokens_seen": 555220992 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200702106318957, + "loss": 2.6002, + "theoretical_loss": 3.871274272328666, + "tokens_seen": 555286528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200601805416249, + "loss": 2.8405, + "theoretical_loss": 3.871225949991965, + "tokens_seen": 555352064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 316316, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.795095682144165, + "objective/train/theoretical_loss": 3.8711776349538143, + "objective/train/tokens_used": 575877600, + "theoretical_loss": 3.8711776349538143, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200501504513541, + "loss": 2.7872, + "theoretical_loss": 3.8711776349538143, + "tokens_seen": 555417600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042004012036108324, + "loss": 2.8702, + "theoretical_loss": 3.8711293272122513, + "tokens_seen": 555483136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200300902708125, + "loss": 2.9276, + "theoretical_loss": 3.871081026765313, + "tokens_seen": 555548672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004200200601805416, + "loss": 2.7231, + "theoretical_loss": 3.871032733611038, + "tokens_seen": 555614208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042001003009027084, + "loss": 2.8501, + "theoretical_loss": 3.870984447747465, + "tokens_seen": 555679744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00042, + "loss": 2.8278, + "theoretical_loss": 3.8709361691726336, + "tokens_seen": 555745280 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199899699097292, + "loss": 2.8936, + "theoretical_loss": 3.870887897884584, + "tokens_seen": 555810816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199799398194584, + "loss": 2.6572, + "theoretical_loss": 3.870839633881358, + "tokens_seen": 555876352 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041996990972918756, + "loss": 2.9405, + "theoretical_loss": 3.8707913771609976, + "tokens_seen": 555941888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041995987963891674, + "loss": 2.4867, + "theoretical_loss": 3.8707431277215454, + "tokens_seen": 556007424 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419949849548646, + "loss": 2.8431, + "theoretical_loss": 3.8706948855610435, + "tokens_seen": 556072960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199398194583751, + "loss": 2.947, + "theoretical_loss": 3.8706466506775383, + "tokens_seen": 556138496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041992978936810434, + "loss": 3.0076, + "theoretical_loss": 3.8705984230690733, + "tokens_seen": 556204032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041991975927783347, + "loss": 2.8488, + "theoretical_loss": 3.870550202733695, + "tokens_seen": 556269568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004199097291875627, + "loss": 2.6245, + "theoretical_loss": 3.8705019896694495, + "tokens_seen": 556335104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198996990972919, + "loss": 2.8031, + "theoretical_loss": 3.8704537838743835, + "tokens_seen": 556400640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041988966900702106, + "loss": 2.6974, + "theoretical_loss": 3.870405585346546, + "tokens_seen": 556466176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041987963891675025, + "loss": 2.6182, + "theoretical_loss": 3.8703573940839857, + "tokens_seen": 556531712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198696088264795, + "loss": 2.6669, + "theoretical_loss": 3.8703092100847516, + "tokens_seen": 556597248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198595787362086, + "loss": 2.8824, + "theoretical_loss": 3.8702610333468943, + "tokens_seen": 556662784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041984954864593784, + "loss": 2.8379, + "theoretical_loss": 3.8702128638684643, + "tokens_seen": 556728320 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041983951855566697, + "loss": 2.602, + "theoretical_loss": 3.870164701647514, + "tokens_seen": 556793856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198294884653962, + "loss": 2.84, + "theoretical_loss": 3.8701165466820955, + "tokens_seen": 556859392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004198194583751254, + "loss": 2.7329, + "theoretical_loss": 3.870068398970262, + "tokens_seen": 556924928 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041980942828485457, + "loss": 2.7621, + "theoretical_loss": 3.870020258510068, + "tokens_seen": 556990464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 317467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9780025482177734, + "objective/train/theoretical_loss": 3.869972125299568, + "objective/train/tokens_used": 577516000, + "theoretical_loss": 3.869972125299568, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041979939819458375, + "loss": 2.8299, + "theoretical_loss": 3.869972125299568, + "tokens_seen": 557056000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041978936810431293, + "loss": 2.7222, + "theoretical_loss": 3.8699239993368177, + "tokens_seen": 557121536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197793380140421, + "loss": 2.6026, + "theoretical_loss": 3.869875880619873, + "tokens_seen": 557187072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041976930792377135, + "loss": 2.8492, + "theoretical_loss": 3.869827769146791, + "tokens_seen": 557252608 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041975927783350047, + "loss": 2.6935, + "theoretical_loss": 3.8697796649156295, + "tokens_seen": 557318144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004197492477432297, + "loss": 2.5823, + "theoretical_loss": 3.8697315679244477, + "tokens_seen": 557383680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041973921765295883, + "loss": 2.7691, + "theoretical_loss": 3.8696834781713036, + "tokens_seen": 557449216 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041972918756268807, + "loss": 2.6545, + "theoretical_loss": 3.8696353956542584, + "tokens_seen": 557514752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041971915747241725, + "loss": 2.6819, + "theoretical_loss": 3.869587320371372, + "tokens_seen": 557580288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041970912738214643, + "loss": 2.7852, + "theoretical_loss": 3.869539252320707, + "tokens_seen": 557645824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196990972918756, + "loss": 2.6624, + "theoretical_loss": 3.8694911915003245, + "tokens_seen": 557711360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041968906720160485, + "loss": 2.8304, + "theoretical_loss": 3.869443137908288, + "tokens_seen": 557776896 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419679037111334, + "loss": 2.9432, + "theoretical_loss": 3.869395091542661, + "tokens_seen": 557842432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196690070210632, + "loss": 2.6705, + "theoretical_loss": 3.8693470524015083, + "tokens_seen": 557907968 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196589769307924, + "loss": 2.9572, + "theoretical_loss": 3.869299020482895, + "tokens_seen": 557973504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196489468405216, + "loss": 2.5857, + "theoretical_loss": 3.8692509957848875, + "tokens_seen": 558039040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196389167502508, + "loss": 2.8012, + "theoretical_loss": 3.8692029783055517, + "tokens_seen": 558104576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041962888665997994, + "loss": 2.8241, + "theoretical_loss": 3.869154968042956, + "tokens_seen": 558170112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041961885656970917, + "loss": 2.8948, + "theoretical_loss": 3.8691069649951677, + "tokens_seen": 558235648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004196088264794383, + "loss": 2.827, + "theoretical_loss": 3.869058969160257, + "tokens_seen": 558301184 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041959879638916753, + "loss": 2.8092, + "theoretical_loss": 3.869010980536292, + "tokens_seen": 558366720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195887662988967, + "loss": 2.8753, + "theoretical_loss": 3.868962999121344, + "tokens_seen": 558432256 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195787362086259, + "loss": 2.8558, + "theoretical_loss": 3.868915024913485, + "tokens_seen": 558497792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195687061183551, + "loss": 2.8519, + "theoretical_loss": 3.8688670579107853, + "tokens_seen": 558563328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195586760280843, + "loss": 2.9895, + "theoretical_loss": 3.8688190981113193, + "tokens_seen": 558628864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 318032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9913525581359863, + "objective/train/theoretical_loss": 3.868771145513159, + "objective/train/tokens_used": 579154400, + "theoretical_loss": 3.868771145513159, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041954864593781344, + "loss": 2.6942, + "theoretical_loss": 3.868771145513159, + "tokens_seen": 558694400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195386158475427, + "loss": 2.7493, + "theoretical_loss": 3.8687232001143794, + "tokens_seen": 558759936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195285857572718, + "loss": 2.9329, + "theoretical_loss": 3.8686752619130553, + "tokens_seen": 558825472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041951855566700104, + "loss": 2.8755, + "theoretical_loss": 3.8686273309072616, + "tokens_seen": 558891008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004195085255767302, + "loss": 2.7809, + "theoretical_loss": 3.8685794070950754, + "tokens_seen": 558956544 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194984954864594, + "loss": 3.0322, + "theoretical_loss": 3.8685314904745742, + "tokens_seen": 559022080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194884653961886, + "loss": 2.6098, + "theoretical_loss": 3.8684835810438347, + "tokens_seen": 559087616 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041947843530591776, + "loss": 2.7776, + "theoretical_loss": 3.868435678800937, + "tokens_seen": 559153152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041946840521564694, + "loss": 2.7414, + "theoretical_loss": 3.868387783743959, + "tokens_seen": 559218688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194583751253762, + "loss": 2.883, + "theoretical_loss": 3.8683398958709816, + "tokens_seen": 559284224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194483450351053, + "loss": 2.6664, + "theoretical_loss": 3.868292015180085, + "tokens_seen": 559349760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041943831494483454, + "loss": 2.7937, + "theoretical_loss": 3.8682441416693516, + "tokens_seen": 559415296 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041942828485456367, + "loss": 2.8014, + "theoretical_loss": 3.868196275336863, + "tokens_seen": 559480832 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194182547642929, + "loss": 2.8912, + "theoretical_loss": 3.8681484161807025, + "tokens_seen": 559546368 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004194082246740221, + "loss": 2.8199, + "theoretical_loss": 3.8681005641989543, + "tokens_seen": 559611904 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041939819458375126, + "loss": 2.792, + "theoretical_loss": 3.8680527193897016, + "tokens_seen": 559677440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041938816449348045, + "loss": 2.8252, + "theoretical_loss": 3.8680048817510313, + "tokens_seen": 559742976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193781344032097, + "loss": 2.9691, + "theoretical_loss": 3.867957051281028, + "tokens_seen": 559808512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193681043129388, + "loss": 2.7795, + "theoretical_loss": 3.867909227977779, + "tokens_seen": 559874048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041935807422266804, + "loss": 2.7819, + "theoretical_loss": 3.8678614118393724, + "tokens_seen": 559939584 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041934804413239717, + "loss": 2.6689, + "theoretical_loss": 3.8678136028638948, + "tokens_seen": 560005120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193380140421264, + "loss": 2.8671, + "theoretical_loss": 3.867765801049437, + "tokens_seen": 560070656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004193279839518556, + "loss": 2.8617, + "theoretical_loss": 3.8677180063940866, + "tokens_seen": 560136192 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041931795386158477, + "loss": 2.8101, + "theoretical_loss": 3.8676702188959355, + "tokens_seen": 560201728 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041930792377131395, + "loss": 2.8684, + "theoretical_loss": 3.8676224385530746, + "tokens_seen": 560267264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 319260, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.651021718978882, + "objective/train/theoretical_loss": 3.867574665363595, + "objective/train/tokens_used": 580792800, + "theoretical_loss": 3.867574665363595, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041929789368104313, + "loss": 2.8879, + "theoretical_loss": 3.867574665363595, + "tokens_seen": 560332800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192878635907723, + "loss": 2.8662, + "theoretical_loss": 3.8675268993255902, + "tokens_seen": 560398336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041927783350050155, + "loss": 2.8046, + "theoretical_loss": 3.867479140437153, + "tokens_seen": 560463872 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041926780341023067, + "loss": 2.7195, + "theoretical_loss": 3.867431388696377, + "tokens_seen": 560529408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192577733199599, + "loss": 2.6808, + "theoretical_loss": 3.867383644101358, + "tokens_seen": 560594944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041924774322968904, + "loss": 2.8332, + "theoretical_loss": 3.8673359066501902, + "tokens_seen": 560660480 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041923771313941827, + "loss": 2.884, + "theoretical_loss": 3.8672881763409714, + "tokens_seen": 560726016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041922768304914745, + "loss": 2.8302, + "theoretical_loss": 3.8672404531717977, + "tokens_seen": 560791552 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041921765295887663, + "loss": 2.9092, + "theoretical_loss": 3.8671927371407664, + "tokens_seen": 560857088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004192076228686058, + "loss": 2.8494, + "theoretical_loss": 3.8671450282459765, + "tokens_seen": 560922624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041919759277833505, + "loss": 2.8229, + "theoretical_loss": 3.867097326485527, + "tokens_seen": 560988160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191875626880642, + "loss": 2.8452, + "theoretical_loss": 3.867049631857518, + "tokens_seen": 561053696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191775325977934, + "loss": 2.8199, + "theoretical_loss": 3.86700194436005, + "tokens_seen": 561119232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041916750250752254, + "loss": 2.9338, + "theoretical_loss": 3.8669542639912238, + "tokens_seen": 561184768 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191574724172518, + "loss": 3.0957, + "theoretical_loss": 3.866906590749142, + "tokens_seen": 561250304 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041914744232698095, + "loss": 2.7999, + "theoretical_loss": 3.8668589246319067, + "tokens_seen": 561315840 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041913741223671014, + "loss": 2.9401, + "theoretical_loss": 3.8668112656376223, + "tokens_seen": 561381376 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191273821464393, + "loss": 2.7745, + "theoretical_loss": 3.8667636137643937, + "tokens_seen": 561446912 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191173520561685, + "loss": 2.9357, + "theoretical_loss": 3.8667159690103237, + "tokens_seen": 561512448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004191073219658977, + "loss": 2.6896, + "theoretical_loss": 3.8666683313735195, + "tokens_seen": 561577984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190972918756269, + "loss": 2.7651, + "theoretical_loss": 3.8666207008520868, + "tokens_seen": 561643520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041908726178535604, + "loss": 2.8477, + "theoretical_loss": 3.8665730774441336, + "tokens_seen": 561709056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190772316950853, + "loss": 3.0061, + "theoretical_loss": 3.866525461147767, + "tokens_seen": 561774592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190672016048144, + "loss": 2.7424, + "theoretical_loss": 3.866477851961096, + "tokens_seen": 561840128 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041905717151454364, + "loss": 2.9924, + "theoretical_loss": 3.8664302498822294, + "tokens_seen": 561905664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 319818, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6618897914886475, + "objective/train/theoretical_loss": 3.8663826549092777, + "objective/train/tokens_used": 582431200, + "theoretical_loss": 3.8663826549092777, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190471414242728, + "loss": 3.0017, + "theoretical_loss": 3.8663826549092777, + "tokens_seen": 561971200 + }, + { + "epoch": 0.17, + "learning_rate": 0.000419037111334002, + "loss": 2.7453, + "theoretical_loss": 3.8663350670403513, + "tokens_seen": 562036736 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190270812437312, + "loss": 2.7955, + "theoretical_loss": 3.8662874862735626, + "tokens_seen": 562102272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004190170511534604, + "loss": 2.7434, + "theoretical_loss": 3.8662399126070226, + "tokens_seen": 562167808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041900702106318954, + "loss": 2.8103, + "theoretical_loss": 3.866192346038845, + "tokens_seen": 562233344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189969909729188, + "loss": 2.7913, + "theoretical_loss": 3.866144786567143, + "tokens_seen": 562298880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189869608826479, + "loss": 2.6216, + "theoretical_loss": 3.8660972341900313, + "tokens_seen": 562364416 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041897693079237714, + "loss": 2.7139, + "theoretical_loss": 3.8660496889056244, + "tokens_seen": 562429952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189669007021063, + "loss": 3.0612, + "theoretical_loss": 3.8660021507120392, + "tokens_seen": 562495488 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189568706118355, + "loss": 2.7447, + "theoretical_loss": 3.8659546196073906, + "tokens_seen": 562561024 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189468405215647, + "loss": 2.624, + "theoretical_loss": 3.8659070955897974, + "tokens_seen": 562626560 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041893681043129387, + "loss": 2.5809, + "theoretical_loss": 3.865859578657377, + "tokens_seen": 562692096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041892678034102305, + "loss": 2.6602, + "theoretical_loss": 3.865812068808248, + "tokens_seen": 562757632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004189167502507523, + "loss": 2.7531, + "theoretical_loss": 3.86576456604053, + "tokens_seen": 562823168 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041890672016048146, + "loss": 2.8401, + "theoretical_loss": 3.8657170703523427, + "tokens_seen": 562888704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041889669007021065, + "loss": 2.7188, + "theoretical_loss": 3.865669581741807, + "tokens_seen": 562954240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188866599799399, + "loss": 2.7441, + "theoretical_loss": 3.865622100207045, + "tokens_seen": 563019776 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418876629889669, + "loss": 2.7755, + "theoretical_loss": 3.8655746257461785, + "tokens_seen": 563085312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041886659979939824, + "loss": 2.8061, + "theoretical_loss": 3.8655271583573305, + "tokens_seen": 563150848 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041885656970912737, + "loss": 2.8223, + "theoretical_loss": 3.8654796980386252, + "tokens_seen": 563216384 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188465396188566, + "loss": 2.7483, + "theoretical_loss": 3.8654322447881864, + "tokens_seen": 563281920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004188365095285858, + "loss": 3.0129, + "theoretical_loss": 3.8653847986041394, + "tokens_seen": 563347456 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041882647943831497, + "loss": 2.9452, + "theoretical_loss": 3.8653373594846103, + "tokens_seen": 563412992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041881644934804415, + "loss": 2.8245, + "theoretical_loss": 3.8652899274277255, + "tokens_seen": 563478528 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041880641925777333, + "loss": 3.1132, + "theoretical_loss": 3.865242502431612, + "tokens_seen": 563544064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 321068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.633208751678467, + "objective/train/theoretical_loss": 3.865195084494398, + "objective/train/tokens_used": 584069600, + "theoretical_loss": 3.865195084494398, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187963891675025, + "loss": 2.6685, + "theoretical_loss": 3.865195084494398, + "tokens_seen": 563609600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041878635907723175, + "loss": 2.5112, + "theoretical_loss": 3.8651476736142127, + "tokens_seen": 563675136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187763289869609, + "loss": 2.7993, + "theoretical_loss": 3.8651002697891848, + "tokens_seen": 563740672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004187662988966901, + "loss": 2.7206, + "theoretical_loss": 3.8650528730174454, + "tokens_seen": 563806208 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041875626880641924, + "loss": 2.8497, + "theoretical_loss": 3.8650054832971237, + "tokens_seen": 563871744 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041874623871614847, + "loss": 2.6861, + "theoretical_loss": 3.8649581006263527, + "tokens_seen": 563937280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041873620862587765, + "loss": 2.8536, + "theoretical_loss": 3.8649107250032646, + "tokens_seen": 564002816 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041872617853560683, + "loss": 2.6021, + "theoretical_loss": 3.8648633564259915, + "tokens_seen": 564068352 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418716148445336, + "loss": 2.8927, + "theoretical_loss": 3.864815994892668, + "tokens_seen": 564133888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041870611835506525, + "loss": 2.6526, + "theoretical_loss": 3.8647686404014276, + "tokens_seen": 564199424 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186960882647944, + "loss": 2.9926, + "theoretical_loss": 3.8647212929504065, + "tokens_seen": 564264960 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186860581745236, + "loss": 2.8074, + "theoretical_loss": 3.8646739525377396, + "tokens_seen": 564330496 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041867602808425274, + "loss": 2.8183, + "theoretical_loss": 3.864626619161564, + "tokens_seen": 564396032 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418665997993982, + "loss": 3.0095, + "theoretical_loss": 3.8645792928200167, + "tokens_seen": 564461568 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041865596790371115, + "loss": 2.9241, + "theoretical_loss": 3.8645319735112356, + "tokens_seen": 564527104 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041864593781344034, + "loss": 2.8379, + "theoretical_loss": 3.8644846612333597, + "tokens_seen": 564592640 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186359077231695, + "loss": 2.9107, + "theoretical_loss": 3.864437355984528, + "tokens_seen": 564658176 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186258776328987, + "loss": 2.6939, + "theoretical_loss": 3.864390057762881, + "tokens_seen": 564723712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186158475426279, + "loss": 2.601, + "theoretical_loss": 3.8643427665665593, + "tokens_seen": 564789248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004186058174523571, + "loss": 2.9561, + "theoretical_loss": 3.864295482393704, + "tokens_seen": 564854784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041859578736208624, + "loss": 2.6425, + "theoretical_loss": 3.8642482052424576, + "tokens_seen": 564920320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185857572718155, + "loss": 2.8036, + "theoretical_loss": 3.864200935110963, + "tokens_seen": 564985856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185757271815446, + "loss": 2.6034, + "theoretical_loss": 3.8641536719973644, + "tokens_seen": 565051392 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041856569709127384, + "loss": 2.6046, + "theoretical_loss": 3.8641064158998057, + "tokens_seen": 565116928 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418555667001003, + "loss": 2.9108, + "theoretical_loss": 3.864059166816431, + "tokens_seen": 565182464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 321728, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3612844944000244, + "objective/train/theoretical_loss": 3.8640119247453875, + "objective/train/tokens_used": 585708000, + "theoretical_loss": 3.8640119247453875, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185456369107322, + "loss": 2.8488, + "theoretical_loss": 3.8640119247453875, + "tokens_seen": 565248000 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185356068204614, + "loss": 2.7459, + "theoretical_loss": 3.863964689684821, + "tokens_seen": 565313536 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004185255767301906, + "loss": 2.7467, + "theoretical_loss": 3.8639174616328784, + "tokens_seen": 565379072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041851554663991974, + "loss": 2.5563, + "theoretical_loss": 3.863870240587708, + "tokens_seen": 565444608 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418505516549649, + "loss": 2.7328, + "theoretical_loss": 3.8638230265474585, + "tokens_seen": 565510144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184954864593781, + "loss": 2.6208, + "theoretical_loss": 3.863775819510278, + "tokens_seen": 565575680 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041848545636910734, + "loss": 2.6409, + "theoretical_loss": 3.863728619474318, + "tokens_seen": 565641216 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184754262788365, + "loss": 2.8141, + "theoretical_loss": 3.8636814264377284, + "tokens_seen": 565706752 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184653961885657, + "loss": 2.6808, + "theoretical_loss": 3.8636342403986603, + "tokens_seen": 565772288 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184553660982949, + "loss": 2.676, + "theoretical_loss": 3.8635870613552665, + "tokens_seen": 565837824 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041844533600802407, + "loss": 2.9051, + "theoretical_loss": 3.863539889305699, + "tokens_seen": 565903360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041843530591775325, + "loss": 2.8133, + "theoretical_loss": 3.863492724248112, + "tokens_seen": 565968896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184252758274825, + "loss": 2.7888, + "theoretical_loss": 3.8634455661806593, + "tokens_seen": 566034432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004184152457372116, + "loss": 2.9215, + "theoretical_loss": 3.8633984151014955, + "tokens_seen": 566099968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041840521564694085, + "loss": 2.8133, + "theoretical_loss": 3.863351271008777, + "tokens_seen": 566165504 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041839518555666997, + "loss": 2.8181, + "theoretical_loss": 3.863304133900659, + "tokens_seen": 566231040 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183851554663992, + "loss": 2.6397, + "theoretical_loss": 3.863257003775299, + "tokens_seen": 566296576 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183751253761284, + "loss": 3.028, + "theoretical_loss": 3.8632098806308557, + "tokens_seen": 566362112 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041836509528585757, + "loss": 2.5726, + "theoretical_loss": 3.8631627644654856, + "tokens_seen": 566427648 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041835506519558675, + "loss": 2.7655, + "theoretical_loss": 3.8631156552773493, + "tokens_seen": 566493184 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418345035105316, + "loss": 2.7821, + "theoretical_loss": 3.8630685530646054, + "tokens_seen": 566558720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183350050150451, + "loss": 2.9222, + "theoretical_loss": 3.8630214578254156, + "tokens_seen": 566624256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041832497492477435, + "loss": 2.7699, + "theoretical_loss": 3.8629743695579393, + "tokens_seen": 566689792 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183149448345035, + "loss": 2.8905, + "theoretical_loss": 3.86292728826034, + "tokens_seen": 566755328 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004183049147442327, + "loss": 2.8477, + "theoretical_loss": 3.8628802139307803, + "tokens_seen": 566820864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 322867, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6280651092529297, + "objective/train/theoretical_loss": 3.8628331465674224, + "objective/train/tokens_used": 587346400, + "theoretical_loss": 3.8628331465674224, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182948846539619, + "loss": 2.8288, + "theoretical_loss": 3.8628331465674224, + "tokens_seen": 566886400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182848545636911, + "loss": 2.683, + "theoretical_loss": 3.8627860861684304, + "tokens_seen": 566951936 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041827482447342025, + "loss": 2.6811, + "theoretical_loss": 3.8627390327319695, + "tokens_seen": 567017472 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041826479438314944, + "loss": 2.9301, + "theoretical_loss": 3.8626919862562055, + "tokens_seen": 567083008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182547642928786, + "loss": 2.6402, + "theoretical_loss": 3.8626449467393034, + "tokens_seen": 567148544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041824473420260785, + "loss": 2.9405, + "theoretical_loss": 3.86259791417943, + "tokens_seen": 567214080 + }, + { + "epoch": 0.17, + "learning_rate": 0.000418234704112337, + "loss": 2.5639, + "theoretical_loss": 3.862550888574753, + "tokens_seen": 567279616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182246740220662, + "loss": 2.9047, + "theoretical_loss": 3.862503869923441, + "tokens_seen": 567345152 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182146439317954, + "loss": 2.5505, + "theoretical_loss": 3.8624568582236622, + "tokens_seen": 567410688 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004182046138415246, + "loss": 2.625, + "theoretical_loss": 3.8624098534735865, + "tokens_seen": 567476224 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041819458375125376, + "loss": 2.9179, + "theoretical_loss": 3.8623628556713836, + "tokens_seen": 567541760 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041818455366098294, + "loss": 2.8786, + "theoretical_loss": 3.8623158648152254, + "tokens_seen": 567607296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181745235707121, + "loss": 2.6793, + "theoretical_loss": 3.862268880903282, + "tokens_seen": 567672832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041816449348044136, + "loss": 2.6765, + "theoretical_loss": 3.8622219039337273, + "tokens_seen": 567738368 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041815446339017054, + "loss": 3.0911, + "theoretical_loss": 3.8621749339047335, + "tokens_seen": 567803904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181444332998997, + "loss": 2.6648, + "theoretical_loss": 3.862127970814474, + "tokens_seen": 567869440 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181344032096289, + "loss": 2.6876, + "theoretical_loss": 3.8620810146611233, + "tokens_seen": 567934976 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181243731193581, + "loss": 3.0197, + "theoretical_loss": 3.862034065442857, + "tokens_seen": 568000512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004181143430290873, + "loss": 2.8192, + "theoretical_loss": 3.8619871231578498, + "tokens_seen": 568066048 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041810431293881644, + "loss": 2.8597, + "theoretical_loss": 3.8619401878042794, + "tokens_seen": 568131584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180942828485457, + "loss": 2.7689, + "theoretical_loss": 3.861893259380322, + "tokens_seen": 568197120 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180842527582748, + "loss": 2.9388, + "theoretical_loss": 3.8618463378841557, + "tokens_seen": 568262656 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041807422266800404, + "loss": 2.7452, + "theoretical_loss": 3.861799423313959, + "tokens_seen": 568328192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180641925777332, + "loss": 2.7699, + "theoretical_loss": 3.861752515667911, + "tokens_seen": 568393728 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180541624874624, + "loss": 2.7727, + "theoretical_loss": 3.861705614944192, + "tokens_seen": 568459264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 324005, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.827101230621338, + "objective/train/theoretical_loss": 3.8616587211409823, + "objective/train/tokens_used": 588984800, + "theoretical_loss": 3.8616587211409823, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180441323971916, + "loss": 2.78, + "theoretical_loss": 3.8616587211409823, + "tokens_seen": 568524800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180341023069208, + "loss": 2.9464, + "theoretical_loss": 3.861611834256463, + "tokens_seen": 568590336 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041802407221664994, + "loss": 2.7669, + "theoretical_loss": 3.8615649542888164, + "tokens_seen": 568655872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180140421263792, + "loss": 2.9159, + "theoretical_loss": 3.861518081236224, + "tokens_seen": 568721408 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004180040120361083, + "loss": 2.6733, + "theoretical_loss": 3.861471215096871, + "tokens_seen": 568786944 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041799398194583754, + "loss": 2.6195, + "theoretical_loss": 3.86142435586894, + "tokens_seen": 568852480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179839518555667, + "loss": 2.5565, + "theoretical_loss": 3.8613775035506164, + "tokens_seen": 568918016 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179739217652959, + "loss": 2.9737, + "theoretical_loss": 3.8613306581400852, + "tokens_seen": 568983552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179638916750251, + "loss": 2.9827, + "theoretical_loss": 3.861283819635532, + "tokens_seen": 569049088 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041795386158475427, + "loss": 2.5987, + "theoretical_loss": 3.861236988035145, + "tokens_seen": 569114624 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041794383149448345, + "loss": 2.8994, + "theoretical_loss": 3.86119016333711, + "tokens_seen": 569180160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179338014042127, + "loss": 2.9604, + "theoretical_loss": 3.861143345539616, + "tokens_seen": 569245696 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004179237713139418, + "loss": 2.5996, + "theoretical_loss": 3.861096534640852, + "tokens_seen": 569311232 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041791374122367105, + "loss": 2.7902, + "theoretical_loss": 3.8610497306390066, + "tokens_seen": 569376768 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041790371113340017, + "loss": 2.6739, + "theoretical_loss": 3.861002933532271, + "tokens_seen": 569442304 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178936810431294, + "loss": 2.668, + "theoretical_loss": 3.860956143318835, + "tokens_seen": 569507840 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178836509528586, + "loss": 2.9281, + "theoretical_loss": 3.8609093599968913, + "tokens_seen": 569573376 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041787362086258777, + "loss": 2.9559, + "theoretical_loss": 3.8608625835646313, + "tokens_seen": 569638912 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041786359077231695, + "loss": 2.6292, + "theoretical_loss": 3.860815814020248, + "tokens_seen": 569704448 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178535606820462, + "loss": 2.8555, + "theoretical_loss": 3.860769051361935, + "tokens_seen": 569769984 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178435305917753, + "loss": 2.6426, + "theoretical_loss": 3.8607222955878866, + "tokens_seen": 569835520 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041783350050150455, + "loss": 2.7501, + "theoretical_loss": 3.860675546696298, + "tokens_seen": 569901056 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178234704112337, + "loss": 2.9662, + "theoretical_loss": 3.8606288046853647, + "tokens_seen": 569966592 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178134403209629, + "loss": 2.7383, + "theoretical_loss": 3.8605820695532826, + "tokens_seen": 570032128 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004178034102306921, + "loss": 2.6567, + "theoretical_loss": 3.860535341298249, + "tokens_seen": 570097664 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 324627, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.945570468902588, + "objective/train/theoretical_loss": 3.860488619918462, + "objective/train/tokens_used": 590623200, + "theoretical_loss": 3.860488619918462, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177933801404213, + "loss": 2.8017, + "theoretical_loss": 3.860488619918462, + "tokens_seen": 570163200 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041778335005015045, + "loss": 3.0509, + "theoretical_loss": 3.8604419054121193, + "tokens_seen": 570228736 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041777331995987964, + "loss": 2.8242, + "theoretical_loss": 3.86039519777742, + "tokens_seen": 570294272 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177632898696088, + "loss": 2.59, + "theoretical_loss": 3.8603484970125637, + "tokens_seen": 570359808 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041775325977933805, + "loss": 2.8078, + "theoretical_loss": 3.8603018031157514, + "tokens_seen": 570425344 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177432296890672, + "loss": 3.0908, + "theoretical_loss": 3.8602551160851837, + "tokens_seen": 570490880 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177331995987964, + "loss": 2.8318, + "theoretical_loss": 3.8602084359190627, + "tokens_seen": 570556416 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177231695085256, + "loss": 2.9725, + "theoretical_loss": 3.86016176261559, + "tokens_seen": 570621952 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004177131394182548, + "loss": 2.67, + "theoretical_loss": 3.8601150961729696, + "tokens_seen": 570687488 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041770310932798396, + "loss": 2.6319, + "theoretical_loss": 3.8600684365894047, + "tokens_seen": 570753024 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041769307923771314, + "loss": 2.8908, + "theoretical_loss": 3.8600217838631, + "tokens_seen": 570818560 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176830491474423, + "loss": 2.7601, + "theoretical_loss": 3.85997513799226, + "tokens_seen": 570884096 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041767301905717156, + "loss": 2.9639, + "theoretical_loss": 3.859928498975092, + "tokens_seen": 570949632 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176629889669007, + "loss": 2.9724, + "theoretical_loss": 3.859881866809801, + "tokens_seen": 571015168 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176529588766299, + "loss": 2.8134, + "theoretical_loss": 3.859835241494595, + "tokens_seen": 571080704 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041764292878635904, + "loss": 2.7331, + "theoretical_loss": 3.859788623027681, + "tokens_seen": 571146240 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176328986960883, + "loss": 2.6944, + "theoretical_loss": 3.859742011407268, + "tokens_seen": 571211776 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041762286860581746, + "loss": 2.8846, + "theoretical_loss": 3.859695406631565, + "tokens_seen": 571277312 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041761283851554664, + "loss": 2.7068, + "theoretical_loss": 3.8596488086987826, + "tokens_seen": 571342848 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004176028084252758, + "loss": 2.7232, + "theoretical_loss": 3.8596022176071303, + "tokens_seen": 571408384 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417592778335005, + "loss": 2.7892, + "theoretical_loss": 3.8595556333548195, + "tokens_seen": 571473920 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175827482447342, + "loss": 2.4485, + "theoretical_loss": 3.859509055940063, + "tokens_seen": 571539456 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175727181544634, + "loss": 2.5149, + "theoretical_loss": 3.859462485361072, + "tokens_seen": 571604992 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041756268806419255, + "loss": 2.6744, + "theoretical_loss": 3.85941592161606, + "tokens_seen": 571670528 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175526579739218, + "loss": 2.7721, + "theoretical_loss": 3.859369364703242, + "tokens_seen": 571736064 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 325757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.774052619934082, + "objective/train/theoretical_loss": 3.8593228146208314, + "objective/train/tokens_used": 592261600, + "theoretical_loss": 3.8593228146208314, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041754262788365096, + "loss": 2.8318, + "theoretical_loss": 3.8593228146208314, + "tokens_seen": 571801600 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041753259779338015, + "loss": 2.966, + "theoretical_loss": 3.859276271367044, + "tokens_seen": 571867136 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175225677031093, + "loss": 2.7908, + "theoretical_loss": 3.859229734940095, + "tokens_seen": 571932672 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175125376128385, + "loss": 2.5661, + "theoretical_loss": 3.8591832053382014, + "tokens_seen": 571998208 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004175025075225677, + "loss": 2.7081, + "theoretical_loss": 3.8591366825595808, + "tokens_seen": 572063744 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174924774322969, + "loss": 2.621, + "theoretical_loss": 3.85909016660245, + "tokens_seen": 572129280 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041748244734202605, + "loss": 2.8749, + "theoretical_loss": 3.8590436574650293, + "tokens_seen": 572194816 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174724172517553, + "loss": 2.705, + "theoretical_loss": 3.858997155145537, + "tokens_seen": 572260352 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174623871614844, + "loss": 2.7483, + "theoretical_loss": 3.858950659642192, + "tokens_seen": 572325888 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041745235707121365, + "loss": 2.7732, + "theoretical_loss": 3.8589041709532164, + "tokens_seen": 572391424 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041744232698094283, + "loss": 2.5971, + "theoretical_loss": 3.8588576890768307, + "tokens_seen": 572456960 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417432296890672, + "loss": 2.672, + "theoretical_loss": 3.858811214011258, + "tokens_seen": 572522496 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174222668004012, + "loss": 2.8971, + "theoretical_loss": 3.8587647457547183, + "tokens_seen": 572588032 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041741223671013037, + "loss": 2.6925, + "theoretical_loss": 3.8587182843054375, + "tokens_seen": 572653568 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004174022066198596, + "loss": 2.6789, + "theoretical_loss": 3.8586718296616382, + "tokens_seen": 572719104 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173921765295888, + "loss": 2.8008, + "theoretical_loss": 3.858625381821545, + "tokens_seen": 572784640 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041738214643931797, + "loss": 2.7738, + "theoretical_loss": 3.858578940783384, + "tokens_seen": 572850176 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041737211634904715, + "loss": 2.721, + "theoretical_loss": 3.85853250654538, + "tokens_seen": 572915712 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173620862587764, + "loss": 2.7751, + "theoretical_loss": 3.8584860791057602, + "tokens_seen": 572981248 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173520561685055, + "loss": 2.5138, + "theoretical_loss": 3.8584396584627516, + "tokens_seen": 573046784 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041734202607823475, + "loss": 2.7566, + "theoretical_loss": 3.8583932446145828, + "tokens_seen": 573112320 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173319959879639, + "loss": 2.808, + "theoretical_loss": 3.8583468375594814, + "tokens_seen": 573177856 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173219658976931, + "loss": 2.7426, + "theoretical_loss": 3.858300437295677, + "tokens_seen": 573243392 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173119358074223, + "loss": 2.7867, + "theoretical_loss": 3.8582540438214, + "tokens_seen": 573308928 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004173019057171515, + "loss": 2.8751, + "theoretical_loss": 3.85820765713488, + "tokens_seen": 573374464 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 326467, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8871238231658936, + "objective/train/theoretical_loss": 3.858161277234349, + "objective/train/tokens_used": 593900000, + "theoretical_loss": 3.858161277234349, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041729187562688065, + "loss": 2.8817, + "theoretical_loss": 3.858161277234349, + "tokens_seen": 573440000 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041728184553660984, + "loss": 2.7829, + "theoretical_loss": 3.8581149041180387, + "tokens_seen": 573505536 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417271815446339, + "loss": 2.7302, + "theoretical_loss": 3.858068537784182, + "tokens_seen": 573571072 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041726178535606825, + "loss": 2.8625, + "theoretical_loss": 3.8580221782310113, + "tokens_seen": 573636608 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172517552657974, + "loss": 2.8321, + "theoretical_loss": 3.8579758254567613, + "tokens_seen": 573702144 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172417251755266, + "loss": 2.7675, + "theoretical_loss": 3.8579294794596657, + "tokens_seen": 573767680 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004172316950852558, + "loss": 2.8152, + "theoretical_loss": 3.85788314023796, + "tokens_seen": 573833216 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417221664994985, + "loss": 2.8103, + "theoretical_loss": 3.857836807789881, + "tokens_seen": 573898752 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041721163490471416, + "loss": 2.833, + "theoretical_loss": 3.857790482113664, + "tokens_seen": 573964288 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041720160481444334, + "loss": 2.886, + "theoretical_loss": 3.8577441632075473, + "tokens_seen": 574029824 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171915747241725, + "loss": 2.9433, + "theoretical_loss": 3.8576978510697675, + "tokens_seen": 574095360 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041718154463390176, + "loss": 2.8847, + "theoretical_loss": 3.8576515456985634, + "tokens_seen": 574160896 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171715145436309, + "loss": 2.6894, + "theoretical_loss": 3.857605247092175, + "tokens_seen": 574226432 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171614844533601, + "loss": 2.6527, + "theoretical_loss": 3.857558955248842, + "tokens_seen": 574291968 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041715145436308924, + "loss": 2.6139, + "theoretical_loss": 3.8575126701668037, + "tokens_seen": 574357504 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171414242728185, + "loss": 2.9821, + "theoretical_loss": 3.8574663918443024, + "tokens_seen": 574423040 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041713139418254766, + "loss": 2.7302, + "theoretical_loss": 3.857420120279579, + "tokens_seen": 574488576 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041712136409227684, + "loss": 2.6908, + "theoretical_loss": 3.857373855470877, + "tokens_seen": 574554112 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417111334002006, + "loss": 2.8465, + "theoretical_loss": 3.8573275974164387, + "tokens_seen": 574619648 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004171013039117352, + "loss": 2.6284, + "theoretical_loss": 3.8572813461145086, + "tokens_seen": 574685184 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170912738214644, + "loss": 2.7787, + "theoretical_loss": 3.8572351015633304, + "tokens_seen": 574750720 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170812437311936, + "loss": 2.6762, + "theoretical_loss": 3.8571888637611496, + "tokens_seen": 574816256 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041707121364092275, + "loss": 2.9144, + "theoretical_loss": 3.857142632706212, + "tokens_seen": 574881792 + }, + { + "epoch": 0.17, + "learning_rate": 0.000417061183550652, + "loss": 2.8601, + "theoretical_loss": 3.857096408396764, + "tokens_seen": 574947328 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041705115346038116, + "loss": 2.6951, + "theoretical_loss": 3.8570501908310515, + "tokens_seen": 575012864 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 327672, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.893551826477051, + "objective/train/theoretical_loss": 3.857003980007324, + "objective/train/tokens_used": 595538400, + "theoretical_loss": 3.857003980007324, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041704112337011035, + "loss": 2.8592, + "theoretical_loss": 3.857003980007324, + "tokens_seen": 575078400 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170310932798395, + "loss": 2.7133, + "theoretical_loss": 3.8569577759238287, + "tokens_seen": 575143936 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170210631895687, + "loss": 2.7563, + "theoretical_loss": 3.8569115785788153, + "tokens_seen": 575209472 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170110330992979, + "loss": 2.9113, + "theoretical_loss": 3.856865387970533, + "tokens_seen": 575275008 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004170010030090271, + "loss": 2.8067, + "theoretical_loss": 3.8568192040972322, + "tokens_seen": 575340544 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041699097291875625, + "loss": 2.5334, + "theoretical_loss": 3.856773026957164, + "tokens_seen": 575406080 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169809428284855, + "loss": 2.765, + "theoretical_loss": 3.8567268565485806, + "tokens_seen": 575471616 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169709127382146, + "loss": 2.7038, + "theoretical_loss": 3.856680692869733, + "tokens_seen": 575537152 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041696088264794385, + "loss": 2.8842, + "theoretical_loss": 3.856634535918875, + "tokens_seen": 575602688 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041695085255767303, + "loss": 2.7644, + "theoretical_loss": 3.85658838569426, + "tokens_seen": 575668224 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169408224674022, + "loss": 2.7948, + "theoretical_loss": 3.856542242194142, + "tokens_seen": 575733760 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169307923771314, + "loss": 2.7855, + "theoretical_loss": 3.8564961054167766, + "tokens_seen": 575799296 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004169207622868606, + "loss": 2.8138, + "theoretical_loss": 3.8564499753604187, + "tokens_seen": 575864832 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041691073219658975, + "loss": 2.8032, + "theoretical_loss": 3.856403852023325, + "tokens_seen": 575930368 + }, + { + "epoch": 0.17, + "learning_rate": 0.000416900702106319, + "loss": 2.9167, + "theoretical_loss": 3.8563577354037513, + "tokens_seen": 575995904 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168906720160481, + "loss": 3.0506, + "theoretical_loss": 3.8563116254999565, + "tokens_seen": 576061440 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041688064192577735, + "loss": 3.0837, + "theoretical_loss": 3.856265522310198, + "tokens_seen": 576126976 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041687061183550653, + "loss": 2.6939, + "theoretical_loss": 3.8562194258327347, + "tokens_seen": 576192512 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168605817452357, + "loss": 2.6713, + "theoretical_loss": 3.856173336065826, + "tokens_seen": 576258048 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168505516549649, + "loss": 2.8135, + "theoretical_loss": 3.856127253007732, + "tokens_seen": 576323584 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168405215646941, + "loss": 2.6818, + "theoretical_loss": 3.8560811766567133, + "tokens_seen": 576389120 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041683049147442326, + "loss": 2.9436, + "theoretical_loss": 3.856035107011032, + "tokens_seen": 576454656 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168204613841525, + "loss": 2.7948, + "theoretical_loss": 3.8559890440689495, + "tokens_seen": 576520192 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004168104312938816, + "loss": 2.7917, + "theoretical_loss": 3.855942987828728, + "tokens_seen": 576585728 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041680040120361085, + "loss": 2.6469, + "theoretical_loss": 3.855896938288632, + "tokens_seen": 576651264 + }, + { + "epoch": 0.17, + "objective/train/docs_used": 328706, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.591416597366333, + "objective/train/theoretical_loss": 3.855850895446925, + "objective/train/tokens_used": 597176800, + "theoretical_loss": 3.855850895446925, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041679037111334, + "loss": 2.5328, + "theoretical_loss": 3.855850895446925, + "tokens_seen": 576716800 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167803410230692, + "loss": 2.7952, + "theoretical_loss": 3.8558048593018714, + "tokens_seen": 576782336 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167703109327984, + "loss": 2.6321, + "theoretical_loss": 3.855758829851737, + "tokens_seen": 576847872 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167602808425276, + "loss": 2.7992, + "theoretical_loss": 3.855712807094787, + "tokens_seen": 576913408 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041675025075225676, + "loss": 2.8481, + "theoretical_loss": 3.8556667910292886, + "tokens_seen": 576978944 + }, + { + "epoch": 0.17, + "learning_rate": 0.000416740220661986, + "loss": 2.6563, + "theoretical_loss": 3.855620781653509, + "tokens_seen": 577044480 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167301905717151, + "loss": 2.8306, + "theoretical_loss": 3.8555747789657158, + "tokens_seen": 577110016 + }, + { + "epoch": 0.17, + "learning_rate": 0.00041672016048144436, + "loss": 2.8782, + "theoretical_loss": 3.855528782964177, + "tokens_seen": 577175552 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167101303911735, + "loss": 2.6459, + "theoretical_loss": 3.8554827936471625, + "tokens_seen": 577241088 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004167001003009027, + "loss": 2.6478, + "theoretical_loss": 3.855436811012942, + "tokens_seen": 577306624 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166900702106319, + "loss": 3.059, + "theoretical_loss": 3.8553908350597865, + "tokens_seen": 577372160 + }, + { + "epoch": 0.17, + "learning_rate": 0.0004166800401203611, + "loss": 2.7494, + "theoretical_loss": 3.855344865785966, + "tokens_seen": 577437696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041667001003009026, + "loss": 2.936, + "theoretical_loss": 3.8552989031897527, + "tokens_seen": 577503232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041665997993981944, + "loss": 2.4975, + "theoretical_loss": 3.855252947269419, + "tokens_seen": 577568768 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166499498495487, + "loss": 2.5769, + "theoretical_loss": 3.8552069980232377, + "tokens_seen": 577634304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041663991975927786, + "loss": 2.8171, + "theoretical_loss": 3.855161055449483, + "tokens_seen": 577699840 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041662988966900704, + "loss": 2.7957, + "theoretical_loss": 3.8551151195464284, + "tokens_seen": 577765376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166198595787362, + "loss": 2.8499, + "theoretical_loss": 3.8550691903123493, + "tokens_seen": 577830912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004166098294884654, + "loss": 2.7152, + "theoretical_loss": 3.855023267745522, + "tokens_seen": 577896448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165997993981946, + "loss": 2.7041, + "theoretical_loss": 3.854977351844221, + "tokens_seen": 577961984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165897693079238, + "loss": 2.7119, + "theoretical_loss": 3.8549314426067243, + "tokens_seen": 578027520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041657973921765295, + "loss": 2.9118, + "theoretical_loss": 3.85488554003131, + "tokens_seen": 578093056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165697091273822, + "loss": 2.7783, + "theoretical_loss": 3.854839644116255, + "tokens_seen": 578158592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041655967903711136, + "loss": 2.8074, + "theoretical_loss": 3.8547937548598386, + "tokens_seen": 578224128 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041654964894684055, + "loss": 2.9678, + "theoretical_loss": 3.8547478722603405, + "tokens_seen": 578289664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 329258, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3565433025360107, + "objective/train/theoretical_loss": 3.8547019963160407, + "objective/train/tokens_used": 598815200, + "theoretical_loss": 3.8547019963160407, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165396188565697, + "loss": 2.7488, + "theoretical_loss": 3.8547019963160407, + "tokens_seen": 578355200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165295887662989, + "loss": 2.957, + "theoretical_loss": 3.854656127025219, + "tokens_seen": 578420736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165195586760281, + "loss": 2.5597, + "theoretical_loss": 3.854610264386158, + "tokens_seen": 578486272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004165095285857573, + "loss": 2.8326, + "theoretical_loss": 3.854564408397139, + "tokens_seen": 578551808 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041649949849548645, + "loss": 2.8244, + "theoretical_loss": 3.854518559056445, + "tokens_seen": 578617344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164894684052157, + "loss": 2.7684, + "theoretical_loss": 3.8544727163623587, + "tokens_seen": 578682880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164794383149448, + "loss": 2.878, + "theoretical_loss": 3.8544268803131643, + "tokens_seen": 578748416 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041646940822467405, + "loss": 2.7852, + "theoretical_loss": 3.8543810509071466, + "tokens_seen": 578813952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041645937813440323, + "loss": 2.7624, + "theoretical_loss": 3.8543352281425904, + "tokens_seen": 578879488 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164493480441324, + "loss": 2.6575, + "theoretical_loss": 3.8542894120177817, + "tokens_seen": 578945024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164393179538616, + "loss": 2.8381, + "theoretical_loss": 3.8542436025310067, + "tokens_seen": 579010560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164292878635908, + "loss": 2.6767, + "theoretical_loss": 3.854197799680553, + "tokens_seen": 579076096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041641925777331995, + "loss": 2.9494, + "theoretical_loss": 3.854152003464708, + "tokens_seen": 579141632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004164092276830492, + "loss": 2.8045, + "theoretical_loss": 3.85410621388176, + "tokens_seen": 579207168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163991975927783, + "loss": 2.8555, + "theoretical_loss": 3.854060430929998, + "tokens_seen": 579272704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041638916750250755, + "loss": 3.0175, + "theoretical_loss": 3.854014654607712, + "tokens_seen": 579338240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041637913741223673, + "loss": 2.8226, + "theoretical_loss": 3.8539688849131917, + "tokens_seen": 579403776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163691073219659, + "loss": 3.0263, + "theoretical_loss": 3.853923121844728, + "tokens_seen": 579469312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163590772316951, + "loss": 2.6421, + "theoretical_loss": 3.8538773654006127, + "tokens_seen": 579534848 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163490471414243, + "loss": 2.8617, + "theoretical_loss": 3.8538316155791383, + "tokens_seen": 579600384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041633901705115346, + "loss": 2.6148, + "theoretical_loss": 3.853785872378597, + "tokens_seen": 579665920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163289869608827, + "loss": 2.7086, + "theoretical_loss": 3.853740135797282, + "tokens_seen": 579731456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004163189568706118, + "loss": 2.9555, + "theoretical_loss": 3.8536944058334885, + "tokens_seen": 579796992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041630892678034105, + "loss": 2.782, + "theoretical_loss": 3.85364868248551, + "tokens_seen": 579862528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162988966900702, + "loss": 3.079, + "theoretical_loss": 3.8536029657516426, + "tokens_seen": 579928064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 329612, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.266913890838623, + "objective/train/theoretical_loss": 3.8535572556301823, + "objective/train/tokens_used": 600453600, + "theoretical_loss": 3.8535572556301823, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162888665997994, + "loss": 2.9089, + "theoretical_loss": 3.8535572556301823, + "tokens_seen": 579993600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162788365095286, + "loss": 2.8912, + "theoretical_loss": 3.8535115521194245, + "tokens_seen": 580059136 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162688064192578, + "loss": 2.857, + "theoretical_loss": 3.8534658552176677, + "tokens_seen": 580124672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041625877632898696, + "loss": 2.8663, + "theoretical_loss": 3.853420164923209, + "tokens_seen": 580190208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162487462387162, + "loss": 2.6089, + "theoretical_loss": 3.853374481234348, + "tokens_seen": 580255744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162387161484453, + "loss": 3.0777, + "theoretical_loss": 3.8533288041493825, + "tokens_seen": 580321280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041622868605817456, + "loss": 2.7666, + "theoretical_loss": 3.853283133666613, + "tokens_seen": 580386816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162186559679037, + "loss": 2.8469, + "theoretical_loss": 3.8532374697843395, + "tokens_seen": 580452352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004162086258776329, + "loss": 3.1069, + "theoretical_loss": 3.853191812500863, + "tokens_seen": 580517888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161985957873621, + "loss": 2.5315, + "theoretical_loss": 3.853146161814485, + "tokens_seen": 580583424 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161885656970913, + "loss": 3.1341, + "theoretical_loss": 3.853100517723509, + "tokens_seen": 580648960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041617853560682046, + "loss": 2.8452, + "theoretical_loss": 3.853054880226236, + "tokens_seen": 580714496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041616850551654964, + "loss": 2.642, + "theoretical_loss": 3.8530092493209707, + "tokens_seen": 580780032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161584754262788, + "loss": 2.9064, + "theoretical_loss": 3.8529636250060166, + "tokens_seen": 580845568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041614844533600806, + "loss": 2.8612, + "theoretical_loss": 3.85291800727968, + "tokens_seen": 580911104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161384152457372, + "loss": 3.071, + "theoretical_loss": 3.8528723961402638, + "tokens_seen": 580976640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161283851554664, + "loss": 2.9736, + "theoretical_loss": 3.852826791586076, + "tokens_seen": 581042176 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041611835506519555, + "loss": 2.659, + "theoretical_loss": 3.8527811936154217, + "tokens_seen": 581107712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004161083249749248, + "loss": 2.744, + "theoretical_loss": 3.8527356022266104, + "tokens_seen": 581173248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041609829488465397, + "loss": 2.8711, + "theoretical_loss": 3.852690017417948, + "tokens_seen": 581238784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041608826479438315, + "loss": 3.0193, + "theoretical_loss": 3.8526444391877437, + "tokens_seen": 581304320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041607823470411233, + "loss": 2.7875, + "theoretical_loss": 3.852598867534307, + "tokens_seen": 581369856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041606820461384156, + "loss": 2.6953, + "theoretical_loss": 3.852553302455947, + "tokens_seen": 581435392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160581745235707, + "loss": 3.0129, + "theoretical_loss": 3.8525077439509747, + "tokens_seen": 581500928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160481444332999, + "loss": 2.7702, + "theoretical_loss": 3.8524621920177005, + "tokens_seen": 581566464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 330311, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.916626453399658, + "objective/train/theoretical_loss": 3.852416646654437, + "objective/train/tokens_used": 602092000, + "theoretical_loss": 3.852416646654437, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041603811434302905, + "loss": 2.8318, + "theoretical_loss": 3.852416646654437, + "tokens_seen": 581632000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004160280842527583, + "loss": 2.5899, + "theoretical_loss": 3.8523711078594953, + "tokens_seen": 581697536 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041601805416248747, + "loss": 2.9073, + "theoretical_loss": 3.8523255756311894, + "tokens_seen": 581763072 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041600802407221665, + "loss": 2.9442, + "theoretical_loss": 3.852280049967832, + "tokens_seen": 581828608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041599799398194583, + "loss": 2.9509, + "theoretical_loss": 3.8522345308677375, + "tokens_seen": 581894144 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415987963891675, + "loss": 2.5574, + "theoretical_loss": 3.8521890183292213, + "tokens_seen": 581959680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159779338014042, + "loss": 2.918, + "theoretical_loss": 3.8521435123505974, + "tokens_seen": 582025216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041596790371113343, + "loss": 2.8101, + "theoretical_loss": 3.8520980129301834, + "tokens_seen": 582090752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041595787362086256, + "loss": 2.611, + "theoretical_loss": 3.852052520066295, + "tokens_seen": 582156288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159478435305918, + "loss": 2.7954, + "theoretical_loss": 3.85200703375725, + "tokens_seen": 582221824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159378134403209, + "loss": 2.8191, + "theoretical_loss": 3.851961554001365, + "tokens_seen": 582287360 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041592778335005015, + "loss": 2.8662, + "theoretical_loss": 3.85191608079696, + "tokens_seen": 582352896 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041591775325977934, + "loss": 2.8765, + "theoretical_loss": 3.8518706141423538, + "tokens_seen": 582418432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004159077231695085, + "loss": 2.9141, + "theoretical_loss": 3.851825154035865, + "tokens_seen": 582483968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041589769307923775, + "loss": 2.9011, + "theoretical_loss": 3.8517797004758156, + "tokens_seen": 582549504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041588766298896693, + "loss": 2.7864, + "theoretical_loss": 3.8517342534605254, + "tokens_seen": 582615040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158776328986961, + "loss": 2.6935, + "theoretical_loss": 3.851688812988317, + "tokens_seen": 582680576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158676028084253, + "loss": 2.9189, + "theoretical_loss": 3.8516433790575118, + "tokens_seen": 582746112 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158575727181545, + "loss": 3.0231, + "theoretical_loss": 3.8515979516664327, + "tokens_seen": 582811648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041584754262788366, + "loss": 2.7779, + "theoretical_loss": 3.851552530813403, + "tokens_seen": 582877184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158375125376129, + "loss": 2.7526, + "theoretical_loss": 3.8515071164967476, + "tokens_seen": 582942720 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415827482447342, + "loss": 2.8509, + "theoretical_loss": 3.8514617087147904, + "tokens_seen": 583008256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041581745235707126, + "loss": 2.9512, + "theoretical_loss": 3.8514163074658576, + "tokens_seen": 583073792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004158074222668004, + "loss": 2.8851, + "theoretical_loss": 3.851370912748274, + "tokens_seen": 583139328 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157973921765296, + "loss": 2.9043, + "theoretical_loss": 3.851325524560367, + "tokens_seen": 583204864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 331192, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3419594764709473, + "objective/train/theoretical_loss": 3.851280142900463, + "objective/train/tokens_used": 603730400, + "theoretical_loss": 3.851280142900463, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157873620862588, + "loss": 2.7418, + "theoretical_loss": 3.851280142900463, + "tokens_seen": 583270400 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415777331995988, + "loss": 2.8718, + "theoretical_loss": 3.8512347677668908, + "tokens_seen": 583335936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041576730190571716, + "loss": 2.7871, + "theoretical_loss": 3.851189399157978, + "tokens_seen": 583401472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157572718154464, + "loss": 2.6968, + "theoretical_loss": 3.8511440370720535, + "tokens_seen": 583467008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157472417251755, + "loss": 2.8935, + "theoretical_loss": 3.851098681507448, + "tokens_seen": 583532544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041573721163490476, + "loss": 2.6811, + "theoretical_loss": 3.8510533324624907, + "tokens_seen": 583598080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157271815446339, + "loss": 2.7665, + "theoretical_loss": 3.8510079899355123, + "tokens_seen": 583663616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157171514543631, + "loss": 3.0478, + "theoretical_loss": 3.8509626539248454, + "tokens_seen": 583729152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004157071213640923, + "loss": 2.8275, + "theoretical_loss": 3.8509173244288206, + "tokens_seen": 583794688 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156970912738215, + "loss": 2.7585, + "theoretical_loss": 3.8508720014457722, + "tokens_seen": 583860224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041568706118355066, + "loss": 2.8001, + "theoretical_loss": 3.8508266849740327, + "tokens_seen": 583925760 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041567703109327984, + "loss": 2.9887, + "theoretical_loss": 3.8507813750119353, + "tokens_seen": 583991296 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415667001003009, + "loss": 2.8654, + "theoretical_loss": 3.8507360715578156, + "tokens_seen": 584056832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041565697091273826, + "loss": 2.9364, + "theoretical_loss": 3.8506907746100087, + "tokens_seen": 584122368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156469408224674, + "loss": 2.8208, + "theoretical_loss": 3.85064548416685, + "tokens_seen": 584187904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004156369107321966, + "loss": 3.1207, + "theoretical_loss": 3.850600200226676, + "tokens_seen": 584253440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041562688064192575, + "loss": 2.9591, + "theoretical_loss": 3.8505549227878237, + "tokens_seen": 584318976 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415616850551655, + "loss": 2.8618, + "theoretical_loss": 3.8505096518486304, + "tokens_seen": 584384512 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041560682046138417, + "loss": 2.8048, + "theoretical_loss": 3.8504643874074347, + "tokens_seen": 584450048 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041559679037111335, + "loss": 2.7938, + "theoretical_loss": 3.8504191294625754, + "tokens_seen": 584515584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041558676028084253, + "loss": 3.0779, + "theoretical_loss": 3.850373878012392, + "tokens_seen": 584581120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041557673019057176, + "loss": 2.727, + "theoretical_loss": 3.850328633055224, + "tokens_seen": 584646656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155667001003009, + "loss": 3.0438, + "theoretical_loss": 3.8502833945894124, + "tokens_seen": 584712192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155566700100301, + "loss": 2.9217, + "theoretical_loss": 3.850238162613299, + "tokens_seen": 584777728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041554663991975925, + "loss": 2.9176, + "theoretical_loss": 3.850192937125225, + "tokens_seen": 584843264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 331815, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.829080104827881, + "objective/train/theoretical_loss": 3.850147718123533, + "objective/train/tokens_used": 605368800, + "theoretical_loss": 3.850147718123533, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004155366098294885, + "loss": 2.7848, + "theoretical_loss": 3.850147718123533, + "tokens_seen": 584908800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041552657973921767, + "loss": 2.877, + "theoretical_loss": 3.850102505606566, + "tokens_seen": 584974336 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041551654964894685, + "loss": 2.7364, + "theoretical_loss": 3.8500572995726676, + "tokens_seen": 585039872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041550651955867603, + "loss": 2.891, + "theoretical_loss": 3.850012100020183, + "tokens_seen": 585105408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154964894684052, + "loss": 2.779, + "theoretical_loss": 3.849966906947456, + "tokens_seen": 585170944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154864593781344, + "loss": 2.9868, + "theoretical_loss": 3.8499217203528335, + "tokens_seen": 585236480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041547642928786363, + "loss": 2.7509, + "theoretical_loss": 3.8498765402346597, + "tokens_seen": 585302016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041546639919759276, + "loss": 2.851, + "theoretical_loss": 3.8498313665912827, + "tokens_seen": 585367552 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415456369107322, + "loss": 2.8037, + "theoretical_loss": 3.84978619942105, + "tokens_seen": 585433088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154463390170511, + "loss": 2.8823, + "theoretical_loss": 3.8497410387223088, + "tokens_seen": 585498624 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041543630892678035, + "loss": 3.0639, + "theoretical_loss": 3.849695884493408, + "tokens_seen": 585564160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041542627883650954, + "loss": 2.5332, + "theoretical_loss": 3.849650736732696, + "tokens_seen": 585629696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154162487462387, + "loss": 2.8753, + "theoretical_loss": 3.849605595438524, + "tokens_seen": 585695232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004154062186559679, + "loss": 3.111, + "theoretical_loss": 3.8495604606092417, + "tokens_seen": 585760768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041539618856569713, + "loss": 2.9358, + "theoretical_loss": 3.8495153322432003, + "tokens_seen": 585826304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041538615847542626, + "loss": 2.7838, + "theoretical_loss": 3.849470210338751, + "tokens_seen": 585891840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153761283851555, + "loss": 2.8008, + "theoretical_loss": 3.849425094894246, + "tokens_seen": 585957376 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153660982948846, + "loss": 2.8761, + "theoretical_loss": 3.8493799859080378, + "tokens_seen": 586022912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041535606820461386, + "loss": 2.9351, + "theoretical_loss": 3.8493348833784813, + "tokens_seen": 586088448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041534603811434304, + "loss": 2.8079, + "theoretical_loss": 3.849289787303929, + "tokens_seen": 586153984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153360080240722, + "loss": 2.8135, + "theoretical_loss": 3.8492446976827357, + "tokens_seen": 586219520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153259779338014, + "loss": 2.7149, + "theoretical_loss": 3.8491996145132568, + "tokens_seen": 586285056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004153159478435306, + "loss": 2.7679, + "theoretical_loss": 3.8491545377938485, + "tokens_seen": 586350592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041530591775325976, + "loss": 2.5723, + "theoretical_loss": 3.849109467522867, + "tokens_seen": 586416128 + }, + { + "epoch": 0.18, + "learning_rate": 0.000415295887662989, + "loss": 2.7364, + "theoretical_loss": 3.8490644036986694, + "tokens_seen": 586481664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 333131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1692469120025635, + "objective/train/theoretical_loss": 3.8490193463196127, + "objective/train/tokens_used": 607007200, + "theoretical_loss": 3.8490193463196127, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152858575727181, + "loss": 2.6856, + "theoretical_loss": 3.8490193463196127, + "tokens_seen": 586547200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041527582748244736, + "loss": 2.8743, + "theoretical_loss": 3.848974295384056, + "tokens_seen": 586612736 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152657973921765, + "loss": 2.8334, + "theoretical_loss": 3.8489292508903574, + "tokens_seen": 586678272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152557673019057, + "loss": 2.7919, + "theoretical_loss": 3.848884212836877, + "tokens_seen": 586743808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152457372116349, + "loss": 2.8642, + "theoretical_loss": 3.848839181221974, + "tokens_seen": 586809344 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152357071213641, + "loss": 2.7809, + "theoretical_loss": 3.84879415604401, + "tokens_seen": 586874880 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041522567703109327, + "loss": 2.907, + "theoretical_loss": 3.8487491373013456, + "tokens_seen": 586940416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004152156469408225, + "loss": 2.6461, + "theoretical_loss": 3.848704124992343, + "tokens_seen": 587005952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041520561685055163, + "loss": 2.59, + "theoretical_loss": 3.8486591191153643, + "tokens_seen": 587071488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041519558676028086, + "loss": 2.8257, + "theoretical_loss": 3.848614119668772, + "tokens_seen": 587137024 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041518555667001, + "loss": 2.8721, + "theoretical_loss": 3.8485691266509314, + "tokens_seen": 587202560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151755265797392, + "loss": 3.1213, + "theoretical_loss": 3.8485241400602055, + "tokens_seen": 587268096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151654964894684, + "loss": 2.6248, + "theoretical_loss": 3.848479159894959, + "tokens_seen": 587333632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151554663991976, + "loss": 2.8182, + "theoretical_loss": 3.8484341861535576, + "tokens_seen": 587399168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151454363089268, + "loss": 2.7422, + "theoretical_loss": 3.8483892188343676, + "tokens_seen": 587464704 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041513540621865595, + "loss": 2.9843, + "theoretical_loss": 3.8483442579357554, + "tokens_seen": 587530240 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004151253761283852, + "loss": 2.7539, + "theoretical_loss": 3.8482993034560877, + "tokens_seen": 587595776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041511534603811437, + "loss": 2.876, + "theoretical_loss": 3.848254355393734, + "tokens_seen": 587661312 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041510531594784355, + "loss": 2.7914, + "theoretical_loss": 3.8482094137470604, + "tokens_seen": 587726848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041509528585757273, + "loss": 2.9117, + "theoretical_loss": 3.8481644785144375, + "tokens_seen": 587792384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041508525576730196, + "loss": 2.741, + "theoretical_loss": 3.8481195496942346, + "tokens_seen": 587857920 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150752256770311, + "loss": 2.6959, + "theoretical_loss": 3.848074627284822, + "tokens_seen": 587923456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150651955867603, + "loss": 2.7976, + "theoretical_loss": 3.84802971128457, + "tokens_seen": 587988992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041505516549648945, + "loss": 2.8536, + "theoretical_loss": 3.84798480169185, + "tokens_seen": 588054528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150451354062187, + "loss": 2.9974, + "theoretical_loss": 3.8479398985050346, + "tokens_seen": 588120064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 334325, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.72865891456604, + "objective/train/theoretical_loss": 3.847895001722496, + "objective/train/tokens_used": 608645600, + "theoretical_loss": 3.847895001722496, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041503510531594787, + "loss": 2.8432, + "theoretical_loss": 3.847895001722496, + "tokens_seen": 588185600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041502507522567705, + "loss": 2.6422, + "theoretical_loss": 3.8478501113426073, + "tokens_seen": 588251136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041501504513540623, + "loss": 2.5214, + "theoretical_loss": 3.847805227363743, + "tokens_seen": 588316672 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004150050150451354, + "loss": 2.6265, + "theoretical_loss": 3.8477603497842763, + "tokens_seen": 588382208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149949849548646, + "loss": 2.6646, + "theoretical_loss": 3.8477154786025825, + "tokens_seen": 588447744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041498495486459383, + "loss": 2.7582, + "theoretical_loss": 3.847670613817038, + "tokens_seen": 588513280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041497492477432296, + "loss": 2.8478, + "theoretical_loss": 3.847625755426018, + "tokens_seen": 588578816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149648946840522, + "loss": 2.6564, + "theoretical_loss": 3.8475809034278994, + "tokens_seen": 588644352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149548645937813, + "loss": 2.4809, + "theoretical_loss": 3.8475360578210602, + "tokens_seen": 588709888 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041494483450351055, + "loss": 2.8239, + "theoretical_loss": 3.847491218603878, + "tokens_seen": 588775424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041493480441323974, + "loss": 2.6325, + "theoretical_loss": 3.847446385774731, + "tokens_seen": 588840960 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149247743229689, + "loss": 2.8157, + "theoretical_loss": 3.847401559331998, + "tokens_seen": 588906496 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004149147442326981, + "loss": 2.8234, + "theoretical_loss": 3.8473567392740593, + "tokens_seen": 588972032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041490471414242733, + "loss": 2.6742, + "theoretical_loss": 3.8473119255992954, + "tokens_seen": 589037568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041489468405215646, + "loss": 2.9159, + "theoretical_loss": 3.8472671183060867, + "tokens_seen": 589103104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148846539618857, + "loss": 2.8963, + "theoretical_loss": 3.8472223173928146, + "tokens_seen": 589168640 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148746238716148, + "loss": 2.7451, + "theoretical_loss": 3.8471775228578617, + "tokens_seen": 589234176 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041486459378134406, + "loss": 2.7525, + "theoretical_loss": 3.8471327346996107, + "tokens_seen": 589299712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041485456369107324, + "loss": 2.73, + "theoretical_loss": 3.8470879529164437, + "tokens_seen": 589365248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148445336008024, + "loss": 2.8554, + "theoretical_loss": 3.847043177506746, + "tokens_seen": 589430784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148345035105316, + "loss": 2.9111, + "theoretical_loss": 3.8469984084689015, + "tokens_seen": 589496320 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148244734202608, + "loss": 2.7169, + "theoretical_loss": 3.8469536458012943, + "tokens_seen": 589561856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041481444332998996, + "loss": 2.8186, + "theoretical_loss": 3.8469088895023114, + "tokens_seen": 589627392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004148044132397192, + "loss": 2.8273, + "theoretical_loss": 3.8468641395703385, + "tokens_seen": 589692928 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147943831494483, + "loss": 2.6757, + "theoretical_loss": 3.8468193960037618, + "tokens_seen": 589758464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 334849, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.178940534591675, + "objective/train/theoretical_loss": 3.8467746588009692, + "objective/train/tokens_used": 610284000, + "theoretical_loss": 3.8467746588009692, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041478435305917756, + "loss": 2.6907, + "theoretical_loss": 3.8467746588009692, + "tokens_seen": 589824000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147743229689067, + "loss": 2.798, + "theoretical_loss": 3.846729927960349, + "tokens_seen": 589889536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147642928786359, + "loss": 2.9737, + "theoretical_loss": 3.846685203480289, + "tokens_seen": 589955072 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147542627883651, + "loss": 2.6387, + "theoretical_loss": 3.8466404853591785, + "tokens_seen": 590020608 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147442326980943, + "loss": 2.5792, + "theoretical_loss": 3.8465957735954075, + "tokens_seen": 590086144 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041473420260782347, + "loss": 3.0323, + "theoretical_loss": 3.8465510681873667, + "tokens_seen": 590151680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004147241725175527, + "loss": 2.8036, + "theoretical_loss": 3.8465063691334453, + "tokens_seen": 590217216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041471414242728183, + "loss": 2.8123, + "theoretical_loss": 3.846461676432037, + "tokens_seen": 590282752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041470411233701106, + "loss": 2.9491, + "theoretical_loss": 3.8464169900815324, + "tokens_seen": 590348288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146940822467402, + "loss": 2.687, + "theoretical_loss": 3.8463723100803247, + "tokens_seen": 590413824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146840521564694, + "loss": 2.708, + "theoretical_loss": 3.846327636426807, + "tokens_seen": 590479360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146740220661986, + "loss": 2.7274, + "theoretical_loss": 3.8462829691193727, + "tokens_seen": 590544896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146639919759278, + "loss": 2.809, + "theoretical_loss": 3.8462383081564173, + "tokens_seen": 590610432 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041465396188565697, + "loss": 2.8318, + "theoretical_loss": 3.846193653536335, + "tokens_seen": 590675968 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041464393179538615, + "loss": 2.8892, + "theoretical_loss": 3.846149005257521, + "tokens_seen": 590741504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041463390170511533, + "loss": 2.6609, + "theoretical_loss": 3.8461043633183722, + "tokens_seen": 590807040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041462387161484457, + "loss": 2.5279, + "theoretical_loss": 3.8460597277172845, + "tokens_seen": 590872576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004146138415245737, + "loss": 2.7456, + "theoretical_loss": 3.846015098452656, + "tokens_seen": 590938112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041460381143430293, + "loss": 2.9341, + "theoretical_loss": 3.8459704755228854, + "tokens_seen": 591003648 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145937813440321, + "loss": 2.9464, + "theoretical_loss": 3.8459258589263694, + "tokens_seen": 591069184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145837512537613, + "loss": 2.7979, + "theoretical_loss": 3.8458812486615077, + "tokens_seen": 591134720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145737211634905, + "loss": 2.7745, + "theoretical_loss": 3.8458366447267003, + "tokens_seen": 591200256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041456369107321965, + "loss": 2.7959, + "theoretical_loss": 3.845792047120347, + "tokens_seen": 591265792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041455366098294884, + "loss": 2.8272, + "theoretical_loss": 3.845747455840849, + "tokens_seen": 591331328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041454363089267807, + "loss": 2.9054, + "theoretical_loss": 3.8457028708866074, + "tokens_seen": 591396864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 335803, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.751352071762085, + "objective/train/theoretical_loss": 3.845658292256025, + "objective/train/tokens_used": 611922400, + "theoretical_loss": 3.845658292256025, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145336008024072, + "loss": 2.7783, + "theoretical_loss": 3.845658292256025, + "tokens_seen": 591462400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041452357071213643, + "loss": 2.7238, + "theoretical_loss": 3.8456137199475027, + "tokens_seen": 591527936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041451354062186556, + "loss": 2.6259, + "theoretical_loss": 3.845569153959445, + "tokens_seen": 591593472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004145035105315948, + "loss": 2.8755, + "theoretical_loss": 3.8455245942902554, + "tokens_seen": 591659008 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414493480441324, + "loss": 2.9484, + "theoretical_loss": 3.8454800409383374, + "tokens_seen": 591724544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041448345035105316, + "loss": 2.7721, + "theoretical_loss": 3.845435493902097, + "tokens_seen": 591790080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041447342026078234, + "loss": 2.8939, + "theoretical_loss": 3.8453909531799395, + "tokens_seen": 591855616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144633901705115, + "loss": 2.9146, + "theoretical_loss": 3.84534641877027, + "tokens_seen": 591921152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144533600802407, + "loss": 2.7202, + "theoretical_loss": 3.845301890671496, + "tokens_seen": 591986688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041444332998996994, + "loss": 2.8713, + "theoretical_loss": 3.845257368882024, + "tokens_seen": 592052224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041443329989969906, + "loss": 3.1036, + "theoretical_loss": 3.8452128534002625, + "tokens_seen": 592117760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004144232698094283, + "loss": 2.7427, + "theoretical_loss": 3.8451683442246196, + "tokens_seen": 592183296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041441323971915753, + "loss": 2.8624, + "theoretical_loss": 3.8451238413535043, + "tokens_seen": 592248832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041440320962888666, + "loss": 2.5677, + "theoretical_loss": 3.8450793447853258, + "tokens_seen": 592314368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143931795386159, + "loss": 2.8878, + "theoretical_loss": 3.8450348545184942, + "tokens_seen": 592379904 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414383149448345, + "loss": 2.5672, + "theoretical_loss": 3.84499037055142, + "tokens_seen": 592445440 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041437311935807426, + "loss": 2.7789, + "theoretical_loss": 3.8449458928825155, + "tokens_seen": 592510976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041436308926780344, + "loss": 2.9836, + "theoretical_loss": 3.8449014215101913, + "tokens_seen": 592576512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143530591775326, + "loss": 2.8149, + "theoretical_loss": 3.84485695643286, + "tokens_seen": 592642048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143430290872618, + "loss": 2.8165, + "theoretical_loss": 3.8448124976489346, + "tokens_seen": 592707584 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414332998996991, + "loss": 3.0223, + "theoretical_loss": 3.8447680451568296, + "tokens_seen": 592773120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041432296890672016, + "loss": 2.7377, + "theoretical_loss": 3.844723598954958, + "tokens_seen": 592838656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143129388164494, + "loss": 2.8208, + "theoretical_loss": 3.8446791590417346, + "tokens_seen": 592904192 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004143029087261785, + "loss": 2.7563, + "theoretical_loss": 3.8446347254155753, + "tokens_seen": 592969728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041429287863590776, + "loss": 2.9717, + "theoretical_loss": 3.8445902980748956, + "tokens_seen": 593035264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 336288, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.919393539428711, + "objective/train/theoretical_loss": 3.8445458770181116, + "objective/train/tokens_used": 613560800, + "theoretical_loss": 3.8445458770181116, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142828485456369, + "loss": 2.8637, + "theoretical_loss": 3.8445458770181116, + "tokens_seen": 593100800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142728184553661, + "loss": 2.6485, + "theoretical_loss": 3.844501462243641, + "tokens_seen": 593166336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142627883650953, + "loss": 2.8594, + "theoretical_loss": 3.8444570537499, + "tokens_seen": 593231872 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142527582748245, + "loss": 2.4827, + "theoretical_loss": 3.8444126515353076, + "tokens_seen": 593297408 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041424272818455367, + "loss": 2.7269, + "theoretical_loss": 3.8443682555982837, + "tokens_seen": 593362944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142326980942829, + "loss": 2.8108, + "theoretical_loss": 3.8443238659372456, + "tokens_seen": 593428480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041422266800401203, + "loss": 2.8032, + "theoretical_loss": 3.844279482550614, + "tokens_seen": 593494016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041421263791374126, + "loss": 2.8399, + "theoretical_loss": 3.84423510543681, + "tokens_seen": 593559552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004142026078234704, + "loss": 2.7256, + "theoretical_loss": 3.8441907345942528, + "tokens_seen": 593625088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141925777331996, + "loss": 2.891, + "theoretical_loss": 3.844146370021366, + "tokens_seen": 593690624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141825476429288, + "loss": 2.8849, + "theoretical_loss": 3.8441020117165703, + "tokens_seen": 593756160 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414172517552658, + "loss": 2.8343, + "theoretical_loss": 3.8440576596782887, + "tokens_seen": 593821696 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041416248746238717, + "loss": 2.9734, + "theoretical_loss": 3.844013313904945, + "tokens_seen": 593887232 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041415245737211635, + "loss": 2.7278, + "theoretical_loss": 3.8439689743949628, + "tokens_seen": 593952768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041414242728184553, + "loss": 2.6113, + "theoretical_loss": 3.8439246411467662, + "tokens_seen": 594018304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041413239719157477, + "loss": 2.7153, + "theoretical_loss": 3.8438803141587807, + "tokens_seen": 594083840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141223671013039, + "loss": 2.9235, + "theoretical_loss": 3.8438359934294315, + "tokens_seen": 594149376 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041411233701103313, + "loss": 2.7539, + "theoretical_loss": 3.843791678957145, + "tokens_seen": 594214912 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004141023069207623, + "loss": 2.8045, + "theoretical_loss": 3.8437473707403482, + "tokens_seen": 594280448 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140922768304915, + "loss": 2.9262, + "theoretical_loss": 3.843703068777467, + "tokens_seen": 594345984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140822467402207, + "loss": 2.9616, + "theoretical_loss": 3.843658773066931, + "tokens_seen": 594411520 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041407221664994985, + "loss": 2.8237, + "theoretical_loss": 3.8436144836071673, + "tokens_seen": 594477056 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041406218655967904, + "loss": 2.9555, + "theoretical_loss": 3.8435702003966052, + "tokens_seen": 594542592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041405215646940827, + "loss": 2.7086, + "theoretical_loss": 3.843525923433675, + "tokens_seen": 594608128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140421263791374, + "loss": 2.8386, + "theoretical_loss": 3.8434816527168056, + "tokens_seen": 594673664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 337523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4961068630218506, + "objective/train/theoretical_loss": 3.8434373882444284, + "objective/train/tokens_used": 615199200, + "theoretical_loss": 3.8434373882444284, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041403209628886663, + "loss": 2.7317, + "theoretical_loss": 3.8434373882444284, + "tokens_seen": 594739200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041402206619859576, + "loss": 2.6545, + "theoretical_loss": 3.8433931300149746, + "tokens_seen": 594804736 + }, + { + "epoch": 0.18, + "learning_rate": 0.000414012036108325, + "loss": 2.952, + "theoretical_loss": 3.843348878026876, + "tokens_seen": 594870272 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004140020060180542, + "loss": 2.8919, + "theoretical_loss": 3.843304632278565, + "tokens_seen": 594935808 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041399197592778336, + "loss": 2.9386, + "theoretical_loss": 3.8432603927684745, + "tokens_seen": 595001344 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041398194583751254, + "loss": 2.533, + "theoretical_loss": 3.8432161594950376, + "tokens_seen": 595066880 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139719157472417, + "loss": 2.7428, + "theoretical_loss": 3.8431719324566895, + "tokens_seen": 595132416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139618856569709, + "loss": 2.6911, + "theoretical_loss": 3.8431277116518636, + "tokens_seen": 595197952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041395185556670014, + "loss": 2.8322, + "theoretical_loss": 3.8430834970789953, + "tokens_seen": 595263488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041394182547642926, + "loss": 2.8112, + "theoretical_loss": 3.843039288736521, + "tokens_seen": 595329024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139317953861585, + "loss": 2.7172, + "theoretical_loss": 3.842995086622877, + "tokens_seen": 595394560 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004139217652958877, + "loss": 2.713, + "theoretical_loss": 3.8429508907364998, + "tokens_seen": 595460096 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041391173520561686, + "loss": 2.8304, + "theoretical_loss": 3.842906701075827, + "tokens_seen": 595525632 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041390170511534604, + "loss": 2.942, + "theoretical_loss": 3.8428625176392965, + "tokens_seen": 595591168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138916750250752, + "loss": 2.8326, + "theoretical_loss": 3.8428183404253473, + "tokens_seen": 595656704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138816449348044, + "loss": 2.8756, + "theoretical_loss": 3.8427741694324182, + "tokens_seen": 595722240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041387161484453364, + "loss": 2.6363, + "theoretical_loss": 3.8427300046589488, + "tokens_seen": 595787776 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041386158475426277, + "loss": 2.7768, + "theoretical_loss": 3.8426858461033797, + "tokens_seen": 595853312 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413851554663992, + "loss": 2.6984, + "theoretical_loss": 3.8426416937641514, + "tokens_seen": 595918848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041384152457372113, + "loss": 2.5968, + "theoretical_loss": 3.842597547639706, + "tokens_seen": 595984384 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041383149448345036, + "loss": 2.9943, + "theoretical_loss": 3.842553407728485, + "tokens_seen": 596049920 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041382146439317954, + "loss": 2.6488, + "theoretical_loss": 3.842509274028931, + "tokens_seen": 596115456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138114343029087, + "loss": 2.6736, + "theoretical_loss": 3.8424651465394866, + "tokens_seen": 596180992 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004138014042126379, + "loss": 2.6599, + "theoretical_loss": 3.842421025258596, + "tokens_seen": 596246528 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137913741223671, + "loss": 2.826, + "theoretical_loss": 3.8423769101847034, + "tokens_seen": 596312064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 338188, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9216253757476807, + "objective/train/theoretical_loss": 3.842332801316254, + "objective/train/tokens_used": 616837600, + "theoretical_loss": 3.842332801316254, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041378134403209627, + "loss": 2.8479, + "theoretical_loss": 3.842332801316254, + "tokens_seen": 596377600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137713139418255, + "loss": 2.7266, + "theoretical_loss": 3.842288698651692, + "tokens_seen": 596443136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041376128385155463, + "loss": 2.689, + "theoretical_loss": 3.8422446021894645, + "tokens_seen": 596508672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041375125376128387, + "loss": 2.9293, + "theoretical_loss": 3.842200511928017, + "tokens_seen": 596574208 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041374122367101305, + "loss": 2.5976, + "theoretical_loss": 3.842156427865797, + "tokens_seen": 596639744 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041373119358074223, + "loss": 2.9682, + "theoretical_loss": 3.842112350001252, + "tokens_seen": 596705280 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137211634904714, + "loss": 2.8541, + "theoretical_loss": 3.84206827833283, + "tokens_seen": 596770816 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004137111334002006, + "loss": 2.7331, + "theoretical_loss": 3.84202421285898, + "tokens_seen": 596836352 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041370110330992977, + "loss": 2.9806, + "theoretical_loss": 3.8419801535781515, + "tokens_seen": 596901888 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413691073219659, + "loss": 2.6249, + "theoretical_loss": 3.841936100488793, + "tokens_seen": 596967424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041368104312938813, + "loss": 2.7589, + "theoretical_loss": 3.8418920535893566, + "tokens_seen": 597032960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041367101303911737, + "loss": 2.6685, + "theoretical_loss": 3.841848012878292, + "tokens_seen": 597098496 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041366098294884655, + "loss": 2.8725, + "theoretical_loss": 3.8418039783540507, + "tokens_seen": 597164032 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041365095285857573, + "loss": 2.767, + "theoretical_loss": 3.841759950015086, + "tokens_seen": 597229568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041364092276830497, + "loss": 2.6848, + "theoretical_loss": 3.841715927859849, + "tokens_seen": 597295104 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136308926780341, + "loss": 2.7853, + "theoretical_loss": 3.8416719118867944, + "tokens_seen": 597360640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041362086258776333, + "loss": 2.6599, + "theoretical_loss": 3.841627902094374, + "tokens_seen": 597426176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136108324974925, + "loss": 2.919, + "theoretical_loss": 3.8415838984810433, + "tokens_seen": 597491712 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004136008024072217, + "loss": 2.8007, + "theoretical_loss": 3.841539901045257, + "tokens_seen": 597557248 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135907723169509, + "loss": 2.7802, + "theoretical_loss": 3.841495909785471, + "tokens_seen": 597622784 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041358074222668005, + "loss": 2.7021, + "theoretical_loss": 3.8414519247001397, + "tokens_seen": 597688320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041357071213640924, + "loss": 2.6627, + "theoretical_loss": 3.8414079457877213, + "tokens_seen": 597753856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041356068204613847, + "loss": 2.7435, + "theoretical_loss": 3.8413639730466715, + "tokens_seen": 597819392 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135506519558676, + "loss": 2.9956, + "theoretical_loss": 3.8413200064754487, + "tokens_seen": 597884928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041354062186559683, + "loss": 2.6928, + "theoretical_loss": 3.841276046072511, + "tokens_seen": 597950464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 339419, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.796257972717285, + "objective/train/theoretical_loss": 3.841232091836317, + "objective/train/tokens_used": 618476000, + "theoretical_loss": 3.841232091836317, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041353059177532596, + "loss": 2.649, + "theoretical_loss": 3.841232091836317, + "tokens_seen": 598016000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135205616850552, + "loss": 2.9916, + "theoretical_loss": 3.8411881437653252, + "tokens_seen": 598081536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004135105315947844, + "loss": 2.767, + "theoretical_loss": 3.841144201857997, + "tokens_seen": 598147072 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041350050150451356, + "loss": 2.6961, + "theoretical_loss": 3.8411002661127913, + "tokens_seen": 598212608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041349047141424274, + "loss": 2.5988, + "theoretical_loss": 3.8410563365281702, + "tokens_seen": 598278144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134804413239719, + "loss": 2.4411, + "theoretical_loss": 3.8410124131025944, + "tokens_seen": 598343680 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134704112337011, + "loss": 2.9326, + "theoretical_loss": 3.8409684958345265, + "tokens_seen": 598409216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041346038114343034, + "loss": 2.7858, + "theoretical_loss": 3.840924584722428, + "tokens_seen": 598474752 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041345035105315946, + "loss": 2.6159, + "theoretical_loss": 3.840880679764763, + "tokens_seen": 598540288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134403209628887, + "loss": 2.9933, + "theoretical_loss": 3.8408367809599957, + "tokens_seen": 598605824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134302908726179, + "loss": 2.634, + "theoretical_loss": 3.8407928883065887, + "tokens_seen": 598671360 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041342026078234706, + "loss": 2.5301, + "theoretical_loss": 3.840749001803008, + "tokens_seen": 598736896 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041341023069207624, + "loss": 2.6363, + "theoretical_loss": 3.8407051214477193, + "tokens_seen": 598802432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004134002006018054, + "loss": 2.7874, + "theoretical_loss": 3.8406612472391872, + "tokens_seen": 598867968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133901705115346, + "loss": 2.9248, + "theoretical_loss": 3.8406173791758786, + "tokens_seen": 598933504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041338014042126384, + "loss": 2.8438, + "theoretical_loss": 3.840573517256261, + "tokens_seen": 598999040 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041337011033099297, + "loss": 2.9863, + "theoretical_loss": 3.8405296614788016, + "tokens_seen": 599064576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133600802407222, + "loss": 2.4437, + "theoretical_loss": 3.8404858118419685, + "tokens_seen": 599130112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041335005015045133, + "loss": 2.6434, + "theoretical_loss": 3.84044196834423, + "tokens_seen": 599195648 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041334002006018056, + "loss": 2.7596, + "theoretical_loss": 3.840398130984056, + "tokens_seen": 599261184 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041332998996990975, + "loss": 2.8734, + "theoretical_loss": 3.840354299759916, + "tokens_seen": 599326720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133199598796389, + "loss": 2.8495, + "theoretical_loss": 3.84031047467028, + "tokens_seen": 599392256 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004133099297893681, + "loss": 2.6736, + "theoretical_loss": 3.840266655713619, + "tokens_seen": 599457792 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132998996990973, + "loss": 3.0788, + "theoretical_loss": 3.840222842888405, + "tokens_seen": 599523328 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041328986960882647, + "loss": 3.0841, + "theoretical_loss": 3.840179036193109, + "tokens_seen": 599588864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 340168, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.811565399169922, + "objective/train/theoretical_loss": 3.840135235626204, + "objective/train/tokens_used": 620114400, + "theoretical_loss": 3.840135235626204, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132798395185557, + "loss": 2.6611, + "theoretical_loss": 3.840135235626204, + "tokens_seen": 599654400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041326980942828483, + "loss": 2.7923, + "theoretical_loss": 3.8400914411861624, + "tokens_seen": 599719936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041325977933801407, + "loss": 2.8316, + "theoretical_loss": 3.8400476528714584, + "tokens_seen": 599785472 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041324974924774325, + "loss": 2.8943, + "theoretical_loss": 3.840003870680566, + "tokens_seen": 599851008 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041323971915747243, + "loss": 2.6886, + "theoretical_loss": 3.8399600946119605, + "tokens_seen": 599916544 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132296890672016, + "loss": 2.8353, + "theoretical_loss": 3.8399163246641157, + "tokens_seen": 599982080 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004132196589769308, + "loss": 3.022, + "theoretical_loss": 3.839872560835509, + "tokens_seen": 600047616 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041320962888665997, + "loss": 2.9232, + "theoretical_loss": 3.839828803124615, + "tokens_seen": 600113152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131995987963892, + "loss": 3.0362, + "theoretical_loss": 3.839785051529912, + "tokens_seen": 600178688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041318956870611833, + "loss": 2.633, + "theoretical_loss": 3.839741306049876, + "tokens_seen": 600244224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041317953861584757, + "loss": 2.9153, + "theoretical_loss": 3.839697566682987, + "tokens_seen": 600309760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131695085255767, + "loss": 2.8834, + "theoretical_loss": 3.8396538334277217, + "tokens_seen": 600375296 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041315947843530593, + "loss": 2.8856, + "theoretical_loss": 3.839610106282559, + "tokens_seen": 600440832 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131494483450351, + "loss": 2.9453, + "theoretical_loss": 3.83956638524598, + "tokens_seen": 600506368 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131394182547643, + "loss": 2.6424, + "theoretical_loss": 3.8395226703164638, + "tokens_seen": 600571904 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131293881644935, + "loss": 3.0177, + "theoretical_loss": 3.8394789614924916, + "tokens_seen": 600637440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004131193580742227, + "loss": 2.7729, + "theoretical_loss": 3.8394352587725438, + "tokens_seen": 600702976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041310932798395184, + "loss": 2.9922, + "theoretical_loss": 3.839391562155103, + "tokens_seen": 600768512 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130992978936811, + "loss": 2.7626, + "theoretical_loss": 3.8393478716386507, + "tokens_seen": 600834048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130892678034102, + "loss": 2.7941, + "theoretical_loss": 3.8393041872216704, + "tokens_seen": 600899584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041307923771313944, + "loss": 2.4599, + "theoretical_loss": 3.8392605089026457, + "tokens_seen": 600965120 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130692076228686, + "loss": 2.6901, + "theoretical_loss": 3.83921683668006, + "tokens_seen": 601030656 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130591775325978, + "loss": 2.7902, + "theoretical_loss": 3.8391731705523977, + "tokens_seen": 601096192 + }, + { + "epoch": 0.18, + "learning_rate": 0.000413049147442327, + "loss": 2.6181, + "theoretical_loss": 3.8391295105181436, + "tokens_seen": 601161728 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041303911735205616, + "loss": 2.7678, + "theoretical_loss": 3.8390858565757844, + "tokens_seen": 601227264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 341333, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.842754364013672, + "objective/train/theoretical_loss": 3.8390422087238045, + "objective/train/tokens_used": 621752800, + "theoretical_loss": 3.8390422087238045, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041302908726178534, + "loss": 2.7321, + "theoretical_loss": 3.8390422087238045, + "tokens_seen": 601292800 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130190571715146, + "loss": 2.9798, + "theoretical_loss": 3.8389985669606927, + "tokens_seen": 601358336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004130090270812437, + "loss": 2.5315, + "theoretical_loss": 3.8389549312849343, + "tokens_seen": 601423872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041299899699097294, + "loss": 2.8848, + "theoretical_loss": 3.838911301695018, + "tokens_seen": 601489408 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041298896690070207, + "loss": 2.8289, + "theoretical_loss": 3.8388676781894313, + "tokens_seen": 601554944 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129789368104313, + "loss": 2.8317, + "theoretical_loss": 3.8388240607666635, + "tokens_seen": 601620480 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129689067201605, + "loss": 2.7689, + "theoretical_loss": 3.838780449425204, + "tokens_seen": 601686016 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041295887662988966, + "loss": 2.5929, + "theoretical_loss": 3.8387368441635425, + "tokens_seen": 601751552 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041294884653961884, + "loss": 2.9393, + "theoretical_loss": 3.8386932449801696, + "tokens_seen": 601817088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129388164493481, + "loss": 2.8339, + "theoretical_loss": 3.8386496518735767, + "tokens_seen": 601882624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129287863590772, + "loss": 2.7379, + "theoretical_loss": 3.8386060648422538, + "tokens_seen": 601948160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041291875626880644, + "loss": 2.9387, + "theoretical_loss": 3.8385624838846946, + "tokens_seen": 602013696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004129087261785356, + "loss": 2.8846, + "theoretical_loss": 3.838518908999391, + "tokens_seen": 602079232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128986960882648, + "loss": 2.9549, + "theoretical_loss": 3.8384753401848353, + "tokens_seen": 602144768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041288866599799404, + "loss": 2.8541, + "theoretical_loss": 3.8384317774395225, + "tokens_seen": 602210304 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041287863590772317, + "loss": 2.7912, + "theoretical_loss": 3.8383882207619457, + "tokens_seen": 602275840 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128686058174524, + "loss": 2.635, + "theoretical_loss": 3.838344670150601, + "tokens_seen": 602341376 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041285857572718153, + "loss": 2.9703, + "theoretical_loss": 3.838301125603982, + "tokens_seen": 602406912 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041284854563691076, + "loss": 2.989, + "theoretical_loss": 3.8382575871205855, + "tokens_seen": 602472448 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041283851554663995, + "loss": 2.6135, + "theoretical_loss": 3.8382140546989074, + "tokens_seen": 602537984 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128284854563691, + "loss": 2.7771, + "theoretical_loss": 3.8381705283374457, + "tokens_seen": 602603520 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128184553660983, + "loss": 2.6952, + "theoretical_loss": 3.8381270080346956, + "tokens_seen": 602669056 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004128084252758275, + "loss": 2.5534, + "theoretical_loss": 3.8380834937891573, + "tokens_seen": 602734592 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041279839518555667, + "loss": 2.8246, + "theoretical_loss": 3.8380399855993286, + "tokens_seen": 602800128 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127883650952859, + "loss": 2.5559, + "theoretical_loss": 3.8379964834637073, + "tokens_seen": 602865664 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 342479, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2375457286834717, + "objective/train/theoretical_loss": 3.8379529873807945, + "objective/train/tokens_used": 623391200, + "theoretical_loss": 3.8379529873807945, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041277833500501503, + "loss": 2.7622, + "theoretical_loss": 3.8379529873807945, + "tokens_seen": 602931200 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041276830491474427, + "loss": 2.7673, + "theoretical_loss": 3.8379094973490897, + "tokens_seen": 602996736 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041275827482447345, + "loss": 3.0164, + "theoretical_loss": 3.837866013367093, + "tokens_seen": 603062272 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041274824473420263, + "loss": 2.8215, + "theoretical_loss": 3.8378225354333066, + "tokens_seen": 603127808 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127382146439318, + "loss": 2.6788, + "theoretical_loss": 3.8377790635462308, + "tokens_seen": 603193344 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412728184553661, + "loss": 2.8211, + "theoretical_loss": 3.83773559770437, + "tokens_seen": 603258880 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041271815446339017, + "loss": 2.9036, + "theoretical_loss": 3.8376921379062248, + "tokens_seen": 603324416 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004127081243731194, + "loss": 2.5925, + "theoretical_loss": 3.8376486841502992, + "tokens_seen": 603389952 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041269809428284854, + "loss": 2.7658, + "theoretical_loss": 3.837605236435097, + "tokens_seen": 603455488 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041268806419257777, + "loss": 2.5781, + "theoretical_loss": 3.8375617947591234, + "tokens_seen": 603521024 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126780341023069, + "loss": 2.856, + "theoretical_loss": 3.8375183591208826, + "tokens_seen": 603586560 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041266800401203613, + "loss": 2.5308, + "theoretical_loss": 3.837474929518879, + "tokens_seen": 603652096 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126579739217653, + "loss": 2.6879, + "theoretical_loss": 3.8374315059516206, + "tokens_seen": 603717632 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126479438314945, + "loss": 2.8421, + "theoretical_loss": 3.8373880884176126, + "tokens_seen": 603783168 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126379137412237, + "loss": 2.881, + "theoretical_loss": 3.837344676915362, + "tokens_seen": 603848704 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126278836509529, + "loss": 2.3681, + "theoretical_loss": 3.8373012714433767, + "tokens_seen": 603914240 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041261785356068204, + "loss": 2.6518, + "theoretical_loss": 3.8372578720001647, + "tokens_seen": 603979776 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004126078234704113, + "loss": 2.8608, + "theoretical_loss": 3.8372144785842344, + "tokens_seen": 604045312 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125977933801404, + "loss": 2.5857, + "theoretical_loss": 3.8371710911940955, + "tokens_seen": 604110848 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041258776328986964, + "loss": 2.3453, + "theoretical_loss": 3.8371277098282572, + "tokens_seen": 604176384 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125777331995988, + "loss": 2.8187, + "theoretical_loss": 3.8370843344852297, + "tokens_seen": 604241920 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412567703109328, + "loss": 2.6687, + "theoretical_loss": 3.8370409651635238, + "tokens_seen": 604307456 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125576730190572, + "loss": 2.7338, + "theoretical_loss": 3.8369976018616505, + "tokens_seen": 604372992 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041254764292878636, + "loss": 2.8385, + "theoretical_loss": 3.8369542445781226, + "tokens_seen": 604438528 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041253761283851554, + "loss": 2.6823, + "theoretical_loss": 3.836910893311451, + "tokens_seen": 604504064 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 342997, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.627711534500122, + "objective/train/theoretical_loss": 3.83686754806015, + "objective/train/tokens_used": 625029600, + "theoretical_loss": 3.83686754806015, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125275827482448, + "loss": 2.6497, + "theoretical_loss": 3.83686754806015, + "tokens_seen": 604569600 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004125175526579739, + "loss": 2.9453, + "theoretical_loss": 3.8368242088227316, + "tokens_seen": 604635136 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041250752256770314, + "loss": 2.9436, + "theoretical_loss": 3.836780875597711, + "tokens_seen": 604700672 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041249749247743227, + "loss": 2.6017, + "theoretical_loss": 3.836737548383601, + "tokens_seen": 604766208 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124874623871615, + "loss": 2.9184, + "theoretical_loss": 3.836694227178919, + "tokens_seen": 604831744 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124774322968907, + "loss": 2.8143, + "theoretical_loss": 3.8366509119821774, + "tokens_seen": 604897280 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041246740220661986, + "loss": 2.9792, + "theoretical_loss": 3.836607602791895, + "tokens_seen": 604962816 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041245737211634904, + "loss": 2.7049, + "theoretical_loss": 3.8365642996065867, + "tokens_seen": 605028352 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124473420260783, + "loss": 2.4616, + "theoretical_loss": 3.83652100242477, + "tokens_seen": 605093888 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004124373119358074, + "loss": 2.7036, + "theoretical_loss": 3.836477711244963, + "tokens_seen": 605159424 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041242728184553664, + "loss": 2.7757, + "theoretical_loss": 3.836434426065683, + "tokens_seen": 605224960 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041241725175526577, + "loss": 2.8633, + "theoretical_loss": 3.836391146885449, + "tokens_seen": 605290496 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412407221664995, + "loss": 2.8372, + "theoretical_loss": 3.8363478737027803, + "tokens_seen": 605356032 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123971915747242, + "loss": 2.6962, + "theoretical_loss": 3.836304606516197, + "tokens_seen": 605421568 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041238716148445337, + "loss": 2.8052, + "theoretical_loss": 3.8362613453242185, + "tokens_seen": 605487104 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041237713139418255, + "loss": 2.6351, + "theoretical_loss": 3.8362180901253655, + "tokens_seen": 605552640 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041236710130391173, + "loss": 2.7867, + "theoretical_loss": 3.83617484091816, + "tokens_seen": 605618176 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123570712136409, + "loss": 2.6174, + "theoretical_loss": 3.8361315977011237, + "tokens_seen": 605683712 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041234704112337015, + "loss": 2.62, + "theoretical_loss": 3.836088360472779, + "tokens_seen": 605749248 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041233701103309927, + "loss": 2.7599, + "theoretical_loss": 3.8360451292316475, + "tokens_seen": 605814784 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004123269809428285, + "loss": 2.6258, + "theoretical_loss": 3.8360019039762543, + "tokens_seen": 605880320 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041231695085255763, + "loss": 2.8677, + "theoretical_loss": 3.835958684705122, + "tokens_seen": 605945856 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041230692076228687, + "loss": 2.9551, + "theoretical_loss": 3.835915471416776, + "tokens_seen": 606011392 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041229689067201605, + "loss": 2.7614, + "theoretical_loss": 3.8358722641097405, + "tokens_seen": 606076928 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041228686058174523, + "loss": 2.7635, + "theoretical_loss": 3.8358290627825413, + "tokens_seen": 606142464 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 344327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1000027656555176, + "objective/train/theoretical_loss": 3.835785867433705, + "objective/train/tokens_used": 626668000, + "theoretical_loss": 3.835785867433705, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122768304914744, + "loss": 2.8939, + "theoretical_loss": 3.835785867433705, + "tokens_seen": 606208000 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041226680040120365, + "loss": 2.8328, + "theoretical_loss": 3.8357426780617567, + "tokens_seen": 606273536 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122567703109328, + "loss": 2.9931, + "theoretical_loss": 3.8356994946652248, + "tokens_seen": 606339072 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412246740220662, + "loss": 2.5709, + "theoretical_loss": 3.835656317242636, + "tokens_seen": 606404608 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041223671013039114, + "loss": 2.7535, + "theoretical_loss": 3.835613145792519, + "tokens_seen": 606470144 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004122266800401204, + "loss": 2.6616, + "theoretical_loss": 3.8355699803134016, + "tokens_seen": 606535680 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041221664994984955, + "loss": 2.766, + "theoretical_loss": 3.8355268208038136, + "tokens_seen": 606601216 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041220661985957874, + "loss": 2.7006, + "theoretical_loss": 3.8354836672622845, + "tokens_seen": 606666752 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121965897693079, + "loss": 2.8494, + "theoretical_loss": 3.835440519687345, + "tokens_seen": 606732288 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121865596790371, + "loss": 2.797, + "theoretical_loss": 3.8353973780775243, + "tokens_seen": 606797824 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121765295887663, + "loss": 2.8522, + "theoretical_loss": 3.8353542424313547, + "tokens_seen": 606863360 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121664994984955, + "loss": 2.9709, + "theoretical_loss": 3.835311112747368, + "tokens_seen": 606928896 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121564694082247, + "loss": 2.7044, + "theoretical_loss": 3.8352679890240964, + "tokens_seen": 606994432 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121464393179539, + "loss": 2.9497, + "theoretical_loss": 3.8352248712600723, + "tokens_seen": 607059968 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121364092276831, + "loss": 2.8229, + "theoretical_loss": 3.835181759453829, + "tokens_seen": 607125504 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041212637913741224, + "loss": 2.7877, + "theoretical_loss": 3.8351386536039005, + "tokens_seen": 607191040 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121163490471415, + "loss": 2.8895, + "theoretical_loss": 3.8350955537088214, + "tokens_seen": 607256576 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004121063189568706, + "loss": 2.7407, + "theoretical_loss": 3.835052459767126, + "tokens_seen": 607322112 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041209628886659984, + "loss": 2.8907, + "theoretical_loss": 3.8350093717773497, + "tokens_seen": 607387648 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412086258776329, + "loss": 2.6943, + "theoretical_loss": 3.8349662897380288, + "tokens_seen": 607453184 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120762286860582, + "loss": 2.8111, + "theoretical_loss": 3.8349232136476994, + "tokens_seen": 607518720 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120661985957874, + "loss": 2.6154, + "theoretical_loss": 3.8348801435048987, + "tokens_seen": 607584256 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041205616850551656, + "loss": 2.7947, + "theoretical_loss": 3.834837079308164, + "tokens_seen": 607649792 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041204613841524574, + "loss": 2.4331, + "theoretical_loss": 3.834794021056033, + "tokens_seen": 607715328 + }, + { + "epoch": 0.18, + "learning_rate": 0.000412036108324975, + "loss": 2.5009, + "theoretical_loss": 3.8347509687470445, + "tokens_seen": 607780864 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 345027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9385602474212646, + "objective/train/theoretical_loss": 3.8347079223797373, + "objective/train/tokens_used": 628306400, + "theoretical_loss": 3.8347079223797373, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004120260782347041, + "loss": 2.7218, + "theoretical_loss": 3.8347079223797373, + "tokens_seen": 607846400 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041201604814443334, + "loss": 2.4772, + "theoretical_loss": 3.8346648819526505, + "tokens_seen": 607911936 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041200601805416247, + "loss": 2.6235, + "theoretical_loss": 3.834621847464325, + "tokens_seen": 607977472 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119959879638917, + "loss": 2.7243, + "theoretical_loss": 3.834578818913301, + "tokens_seen": 608043008 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119859578736209, + "loss": 3.0051, + "theoretical_loss": 3.8345357962981197, + "tokens_seen": 608108544 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041197592778335006, + "loss": 2.7735, + "theoretical_loss": 3.834492779617322, + "tokens_seen": 608174080 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041196589769307924, + "loss": 2.628, + "theoretical_loss": 3.834449768869451, + "tokens_seen": 608239616 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119558676028085, + "loss": 2.6384, + "theoretical_loss": 3.834406764053048, + "tokens_seen": 608305152 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119458375125376, + "loss": 2.5853, + "theoretical_loss": 3.834363765166657, + "tokens_seen": 608370688 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041193580742226684, + "loss": 2.8413, + "theoretical_loss": 3.834320772208822, + "tokens_seen": 608436224 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041192577733199597, + "loss": 2.8341, + "theoretical_loss": 3.8342777851780863, + "tokens_seen": 608501760 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119157472417252, + "loss": 2.5505, + "theoretical_loss": 3.8342348040729943, + "tokens_seen": 608567296 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004119057171514544, + "loss": 2.6599, + "theoretical_loss": 3.8341918288920924, + "tokens_seen": 608632832 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041189568706118357, + "loss": 2.7973, + "theoretical_loss": 3.8341488596339257, + "tokens_seen": 608698368 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041188565697091275, + "loss": 2.799, + "theoretical_loss": 3.8341058962970402, + "tokens_seen": 608763904 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041187562688064193, + "loss": 2.8924, + "theoretical_loss": 3.834062938879982, + "tokens_seen": 608829440 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118655967903711, + "loss": 2.6892, + "theoretical_loss": 3.8340199873813, + "tokens_seen": 608894976 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041185556670010035, + "loss": 2.8457, + "theoretical_loss": 3.8339770417995407, + "tokens_seen": 608960512 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041184553660982947, + "loss": 2.9086, + "theoretical_loss": 3.8339341021332527, + "tokens_seen": 609026048 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004118355065195587, + "loss": 2.7818, + "theoretical_loss": 3.833891168380984, + "tokens_seen": 609091584 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041182547642928783, + "loss": 2.6026, + "theoretical_loss": 3.833848240541286, + "tokens_seen": 609157120 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041181544633901707, + "loss": 2.7824, + "theoretical_loss": 3.833805318612706, + "tokens_seen": 609222656 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041180541624874625, + "loss": 2.6473, + "theoretical_loss": 3.8337624025937957, + "tokens_seen": 609288192 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041179538615847543, + "loss": 2.7189, + "theoretical_loss": 3.8337194924831053, + "tokens_seen": 609353728 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117853560682046, + "loss": 2.7765, + "theoretical_loss": 3.833676588279187, + "tokens_seen": 609419264 + }, + { + "epoch": 0.18, + "objective/train/docs_used": 346111, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7798197269439697, + "objective/train/theoretical_loss": 3.8336336899805916, + "objective/train/tokens_used": 629944800, + "theoretical_loss": 3.8336336899805916, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041177532597793385, + "loss": 2.5951, + "theoretical_loss": 3.8336336899805916, + "tokens_seen": 609484800 + }, + { + "epoch": 0.18, + "learning_rate": 0.000411765295887663, + "loss": 2.6398, + "theoretical_loss": 3.833590797585872, + "tokens_seen": 609550336 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117552657973922, + "loss": 2.7884, + "theoretical_loss": 3.833547911093581, + "tokens_seen": 609615872 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041174523570712134, + "loss": 2.7703, + "theoretical_loss": 3.8335050305022715, + "tokens_seen": 609681408 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117352056168506, + "loss": 2.696, + "theoretical_loss": 3.833462155810498, + "tokens_seen": 609746944 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041172517552657975, + "loss": 2.8528, + "theoretical_loss": 3.833419287016815, + "tokens_seen": 609812480 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041171514543630894, + "loss": 2.6289, + "theoretical_loss": 3.8333764241197765, + "tokens_seen": 609878016 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004117051153460381, + "loss": 2.6666, + "theoretical_loss": 3.833333567117939, + "tokens_seen": 609943552 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116950852557673, + "loss": 2.97, + "theoretical_loss": 3.8332907160098575, + "tokens_seen": 610009088 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116850551654965, + "loss": 2.8593, + "theoretical_loss": 3.8332478707940885, + "tokens_seen": 610074624 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116750250752257, + "loss": 2.6773, + "theoretical_loss": 3.8332050314691894, + "tokens_seen": 610140160 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041166499498495484, + "loss": 2.8794, + "theoretical_loss": 3.833162198033718, + "tokens_seen": 610205696 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116549648946841, + "loss": 2.9646, + "theoretical_loss": 3.833119370486231, + "tokens_seen": 610271232 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116449348044132, + "loss": 2.7601, + "theoretical_loss": 3.8330765488252876, + "tokens_seen": 610336768 + }, + { + "epoch": 0.18, + "learning_rate": 0.00041163490471414244, + "loss": 2.6683, + "theoretical_loss": 3.8330337330494473, + "tokens_seen": 610402304 + }, + { + "epoch": 0.18, + "learning_rate": 0.0004116248746238716, + "loss": 2.8002, + "theoretical_loss": 3.8329909231572685, + "tokens_seen": 610467840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004116148445336008, + "loss": 2.7484, + "theoretical_loss": 3.832948119147312, + "tokens_seen": 610533376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041160481444333, + "loss": 2.6991, + "theoretical_loss": 3.8329053210181376, + "tokens_seen": 610598912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115947843530592, + "loss": 2.7888, + "theoretical_loss": 3.8328625287683065, + "tokens_seen": 610664448 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041158475426278834, + "loss": 2.8296, + "theoretical_loss": 3.8328197423963806, + "tokens_seen": 610729984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115747241725176, + "loss": 2.712, + "theoretical_loss": 3.8327769619009215, + "tokens_seen": 610795520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115646940822467, + "loss": 2.8347, + "theoretical_loss": 3.832734187280492, + "tokens_seen": 610861056 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041155466399197594, + "loss": 2.9096, + "theoretical_loss": 3.832691418533655, + "tokens_seen": 610926592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115446339017051, + "loss": 2.4744, + "theoretical_loss": 3.832648655658973, + "tokens_seen": 610992128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115346038114343, + "loss": 2.8143, + "theoretical_loss": 3.8326058986550127, + "tokens_seen": 611057664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 346814, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.812352180480957, + "objective/train/theoretical_loss": 3.832563147520336, + "objective/train/tokens_used": 631583200, + "theoretical_loss": 3.832563147520336, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004115245737211635, + "loss": 2.8388, + "theoretical_loss": 3.832563147520336, + "tokens_seen": 611123200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041151454363089267, + "loss": 2.8298, + "theoretical_loss": 3.8325204022535084, + "tokens_seen": 611188736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041150451354062185, + "loss": 2.8495, + "theoretical_loss": 3.832477662853096, + "tokens_seen": 611254272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114944834503511, + "loss": 2.829, + "theoretical_loss": 3.8324349293176656, + "tokens_seen": 611319808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114844533600802, + "loss": 2.6977, + "theoretical_loss": 3.8323922016457823, + "tokens_seen": 611385344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041147442326980944, + "loss": 2.774, + "theoretical_loss": 3.8323494798360134, + "tokens_seen": 611450880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041146439317953857, + "loss": 2.7721, + "theoretical_loss": 3.832306763886927, + "tokens_seen": 611516416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114543630892678, + "loss": 2.8789, + "theoretical_loss": 3.8322640537970916, + "tokens_seen": 611581952 + }, + { + "epoch": 0.19, + "learning_rate": 0.000411444332998997, + "loss": 2.794, + "theoretical_loss": 3.8322213495650743, + "tokens_seen": 611647488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041143430290872617, + "loss": 2.6454, + "theoretical_loss": 3.8321786511894453, + "tokens_seen": 611713024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041142427281845535, + "loss": 2.9449, + "theoretical_loss": 3.8321359586687738, + "tokens_seen": 611778560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004114142427281846, + "loss": 2.8482, + "theoretical_loss": 3.83209327200163, + "tokens_seen": 611844096 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041140421263791377, + "loss": 2.7537, + "theoretical_loss": 3.832050591186584, + "tokens_seen": 611909632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041139418254764295, + "loss": 2.6385, + "theoretical_loss": 3.832007916222208, + "tokens_seen": 611975168 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041138415245737213, + "loss": 2.9391, + "theoretical_loss": 3.8319652471070724, + "tokens_seen": 612040704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113741223671013, + "loss": 2.7014, + "theoretical_loss": 3.8319225838397495, + "tokens_seen": 612106240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041136409227683055, + "loss": 2.8633, + "theoretical_loss": 3.8318799264188126, + "tokens_seen": 612171776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041135406218655967, + "loss": 2.7242, + "theoretical_loss": 3.831837274842834, + "tokens_seen": 612237312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004113440320962889, + "loss": 2.6845, + "theoretical_loss": 3.8317946291103873, + "tokens_seen": 612302848 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041133400200601803, + "loss": 2.723, + "theoretical_loss": 3.8317519892200473, + "tokens_seen": 612368384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041132397191574727, + "loss": 2.8871, + "theoretical_loss": 3.8317093551703882, + "tokens_seen": 612433920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041131394182547645, + "loss": 2.9039, + "theoretical_loss": 3.8316667269599844, + "tokens_seen": 612499456 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041130391173520563, + "loss": 2.7024, + "theoretical_loss": 3.831624104587413, + "tokens_seen": 612564992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112938816449348, + "loss": 2.6651, + "theoretical_loss": 3.831581488051248, + "tokens_seen": 612630528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041128385155466405, + "loss": 2.7755, + "theoretical_loss": 3.8315388773500683, + "tokens_seen": 612696064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 348068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.487290859222412, + "objective/train/theoretical_loss": 3.8314962724824495, + "objective/train/tokens_used": 633221600, + "theoretical_loss": 3.8314962724824495, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112738214643932, + "loss": 2.6193, + "theoretical_loss": 3.8314962724824495, + "tokens_seen": 612761600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112637913741224, + "loss": 2.9404, + "theoretical_loss": 3.831453673446969, + "tokens_seen": 612827136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041125376128385154, + "loss": 2.6889, + "theoretical_loss": 3.831411080242206, + "tokens_seen": 612892672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112437311935808, + "loss": 2.8746, + "theoretical_loss": 3.831368492866738, + "tokens_seen": 612958208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041123370110330995, + "loss": 2.7783, + "theoretical_loss": 3.831325911319145, + "tokens_seen": 613023744 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041122367101303914, + "loss": 2.529, + "theoretical_loss": 3.8312833355980063, + "tokens_seen": 613089280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112136409227683, + "loss": 2.8101, + "theoretical_loss": 3.831240765701901, + "tokens_seen": 613154816 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004112036108324975, + "loss": 2.5207, + "theoretical_loss": 3.8311982016294115, + "tokens_seen": 613220352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111935807422267, + "loss": 2.7324, + "theoretical_loss": 3.831155643379117, + "tokens_seen": 613285888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111835506519559, + "loss": 2.7735, + "theoretical_loss": 3.831113090949601, + "tokens_seen": 613351424 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041117352056168504, + "loss": 2.91, + "theoretical_loss": 3.8310705443394433, + "tokens_seen": 613416960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111634904714143, + "loss": 3.0031, + "theoretical_loss": 3.8310280035472286, + "tokens_seen": 613482496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111534603811434, + "loss": 2.77, + "theoretical_loss": 3.830985468571538, + "tokens_seen": 613548032 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041114343029087264, + "loss": 2.6051, + "theoretical_loss": 3.8309429394109573, + "tokens_seen": 613613568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111334002006018, + "loss": 2.6809, + "theoretical_loss": 3.8309004160640683, + "tokens_seen": 613679104 + }, + { + "epoch": 0.19, + "learning_rate": 0.000411123370110331, + "loss": 2.5764, + "theoretical_loss": 3.830857898529457, + "tokens_seen": 613744640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111133400200602, + "loss": 2.8588, + "theoretical_loss": 3.8308153868057078, + "tokens_seen": 613810176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004111033099297894, + "loss": 2.6019, + "theoretical_loss": 3.8307728808914066, + "tokens_seen": 613875712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041109327983951854, + "loss": 2.6243, + "theoretical_loss": 3.8307303807851394, + "tokens_seen": 613941248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110832497492478, + "loss": 2.9228, + "theoretical_loss": 3.8306878864854923, + "tokens_seen": 614006784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110732196589769, + "loss": 2.6788, + "theoretical_loss": 3.8306453979910526, + "tokens_seen": 614072320 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041106318956870614, + "loss": 2.7896, + "theoretical_loss": 3.830602915300408, + "tokens_seen": 614137856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110531594784353, + "loss": 2.8886, + "theoretical_loss": 3.8305604384121468, + "tokens_seen": 614203392 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110431293881645, + "loss": 2.6726, + "theoretical_loss": 3.830517967324856, + "tokens_seen": 614268928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110330992978937, + "loss": 2.7219, + "theoretical_loss": 3.8304755020371264, + "tokens_seen": 614334464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 348575, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.63563871383667, + "objective/train/theoretical_loss": 3.830433042547546, + "objective/train/tokens_used": 634860000, + "theoretical_loss": 3.830433042547546, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041102306920762287, + "loss": 2.8635, + "theoretical_loss": 3.830433042547546, + "tokens_seen": 614400000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041101303911735205, + "loss": 2.7305, + "theoretical_loss": 3.8303905888547067, + "tokens_seen": 614465536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004110030090270813, + "loss": 2.823, + "theoretical_loss": 3.830348140957197, + "tokens_seen": 614531072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109929789368104, + "loss": 2.6875, + "theoretical_loss": 3.830305698853609, + "tokens_seen": 614596608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041098294884653965, + "loss": 2.8067, + "theoretical_loss": 3.830263262542534, + "tokens_seen": 614662144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041097291875626877, + "loss": 2.755, + "theoretical_loss": 3.8302208320225635, + "tokens_seen": 614727680 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410962888665998, + "loss": 2.7622, + "theoretical_loss": 3.8301784072922898, + "tokens_seen": 614793216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109528585757272, + "loss": 2.7521, + "theoretical_loss": 3.8301359883503068, + "tokens_seen": 614858752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041094282848545637, + "loss": 2.606, + "theoretical_loss": 3.8300935751952077, + "tokens_seen": 614924288 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041093279839518555, + "loss": 2.7874, + "theoretical_loss": 3.8300511678255855, + "tokens_seen": 614989824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109227683049148, + "loss": 2.6015, + "theoretical_loss": 3.830008766240036, + "tokens_seen": 615055360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004109127382146439, + "loss": 2.6941, + "theoretical_loss": 3.8299663704371527, + "tokens_seen": 615120896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041090270812437315, + "loss": 2.5843, + "theoretical_loss": 3.829923980415532, + "tokens_seen": 615186432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108926780341023, + "loss": 2.8617, + "theoretical_loss": 3.8298815961737693, + "tokens_seen": 615251968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108826479438315, + "loss": 2.7504, + "theoretical_loss": 3.829839217710462, + "tokens_seen": 615317504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108726178535607, + "loss": 2.6976, + "theoretical_loss": 3.8297968450242053, + "tokens_seen": 615383040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041086258776328987, + "loss": 2.874, + "theoretical_loss": 3.829754478113597, + "tokens_seen": 615448576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041085255767301905, + "loss": 2.6128, + "theoretical_loss": 3.829712116977236, + "tokens_seen": 615514112 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041084252758274823, + "loss": 2.6613, + "theoretical_loss": 3.82966976161372, + "tokens_seen": 615579648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108324974924774, + "loss": 2.7108, + "theoretical_loss": 3.8296274120216474, + "tokens_seen": 615645184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041082246740220665, + "loss": 2.5394, + "theoretical_loss": 3.829585068199618, + "tokens_seen": 615710720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004108124373119358, + "loss": 2.7524, + "theoretical_loss": 3.8295427301462324, + "tokens_seen": 615776256 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410802407221665, + "loss": 2.8075, + "theoretical_loss": 3.8295003978600892, + "tokens_seen": 615841792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107923771313942, + "loss": 2.7175, + "theoretical_loss": 3.82945807133979, + "tokens_seen": 615907328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107823470411234, + "loss": 2.7832, + "theoretical_loss": 3.8294157505839364, + "tokens_seen": 615972864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 349861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8283627033233643, + "objective/train/theoretical_loss": 3.8293734355911297, + "objective/train/tokens_used": 636498400, + "theoretical_loss": 3.8293734355911297, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041077231695085256, + "loss": 2.578, + "theoretical_loss": 3.8293734355911297, + "tokens_seen": 616038400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041076228686058174, + "loss": 2.726, + "theoretical_loss": 3.829331126359972, + "tokens_seen": 616103936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107522567703109, + "loss": 2.8416, + "theoretical_loss": 3.8292888228890662, + "tokens_seen": 616169472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041074222668004015, + "loss": 2.7834, + "theoretical_loss": 3.8292465251770165, + "tokens_seen": 616235008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107321965897693, + "loss": 2.5378, + "theoretical_loss": 3.829204233222425, + "tokens_seen": 616300544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107221664994985, + "loss": 2.7511, + "theoretical_loss": 3.829161947023897, + "tokens_seen": 616366080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041071213640922764, + "loss": 2.5905, + "theoretical_loss": 3.829119666580037, + "tokens_seen": 616431616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004107021063189569, + "loss": 2.8605, + "theoretical_loss": 3.82907739188945, + "tokens_seen": 616497152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041069207622868606, + "loss": 2.9454, + "theoretical_loss": 3.8290351229507418, + "tokens_seen": 616562688 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041068204613841524, + "loss": 2.8061, + "theoretical_loss": 3.8289928597625185, + "tokens_seen": 616628224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106720160481444, + "loss": 2.7851, + "theoretical_loss": 3.828950602323386, + "tokens_seen": 616693760 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106619859578736, + "loss": 2.7758, + "theoretical_loss": 3.828908350631953, + "tokens_seen": 616759296 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041065195586760284, + "loss": 2.8322, + "theoretical_loss": 3.828866104686825, + "tokens_seen": 616824832 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410641925777332, + "loss": 2.7427, + "theoretical_loss": 3.828823864486612, + "tokens_seen": 616890368 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106318956870612, + "loss": 2.854, + "theoretical_loss": 3.828781630029922, + "tokens_seen": 616955904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106218655967904, + "loss": 2.7491, + "theoretical_loss": 3.8287394013153637, + "tokens_seen": 617021440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004106118355065196, + "loss": 2.5018, + "theoretical_loss": 3.828697178341547, + "tokens_seen": 617086976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041060180541624874, + "loss": 2.7982, + "theoretical_loss": 3.8286549611070813, + "tokens_seen": 617152512 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410591775325978, + "loss": 2.7268, + "theoretical_loss": 3.8286127496105777, + "tokens_seen": 617218048 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105817452357071, + "loss": 2.8512, + "theoretical_loss": 3.828570543850647, + "tokens_seen": 617283584 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041057171514543634, + "loss": 2.7221, + "theoretical_loss": 3.8285283438259015, + "tokens_seen": 617349120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105616850551655, + "loss": 2.7407, + "theoretical_loss": 3.8284861495349514, + "tokens_seen": 617414656 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105516549648947, + "loss": 2.6172, + "theoretical_loss": 3.828443960976411, + "tokens_seen": 617480192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105416248746239, + "loss": 2.8896, + "theoretical_loss": 3.828401778148892, + "tokens_seen": 617545728 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041053159478435307, + "loss": 2.5109, + "theoretical_loss": 3.828359601051008, + "tokens_seen": 617611264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 350619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7856407165527344, + "objective/train/theoretical_loss": 3.8283174296813733, + "objective/train/tokens_used": 638136800, + "theoretical_loss": 3.8283174296813733, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041052156469408225, + "loss": 2.6453, + "theoretical_loss": 3.8283174296813733, + "tokens_seen": 617676800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105115346038115, + "loss": 2.6044, + "theoretical_loss": 3.828275264038602, + "tokens_seen": 617742336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004105015045135406, + "loss": 2.5576, + "theoretical_loss": 3.828233104121309, + "tokens_seen": 617807872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041049147442326985, + "loss": 2.6203, + "theoretical_loss": 3.8281909499281097, + "tokens_seen": 617873408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041048144433299897, + "loss": 2.6868, + "theoretical_loss": 3.82814880145762, + "tokens_seen": 617938944 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104714142427282, + "loss": 2.7666, + "theoretical_loss": 3.828106658708456, + "tokens_seen": 618004480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104613841524574, + "loss": 2.8267, + "theoretical_loss": 3.8280645216792344, + "tokens_seen": 618070016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041045135406218657, + "loss": 2.9681, + "theoretical_loss": 3.828022390368572, + "tokens_seen": 618135552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041044132397191575, + "loss": 2.6423, + "theoretical_loss": 3.8279802647750882, + "tokens_seen": 618201088 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410431293881645, + "loss": 2.4776, + "theoretical_loss": 3.8279381448973995, + "tokens_seen": 618266624 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104212637913741, + "loss": 2.5911, + "theoretical_loss": 3.827896030734126, + "tokens_seen": 618332160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041041123370110335, + "loss": 2.4597, + "theoretical_loss": 3.8278539222838854, + "tokens_seen": 618397696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004104012036108325, + "loss": 2.8388, + "theoretical_loss": 3.827811819545299, + "tokens_seen": 618463232 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103911735205617, + "loss": 2.7457, + "theoretical_loss": 3.827769722516985, + "tokens_seen": 618528768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103811434302909, + "loss": 2.8415, + "theoretical_loss": 3.8277276311975656, + "tokens_seen": 618594304 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041037111334002007, + "loss": 2.8539, + "theoretical_loss": 3.8276855455856618, + "tokens_seen": 618659840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041036108324974925, + "loss": 2.7644, + "theoretical_loss": 3.8276434656798948, + "tokens_seen": 618725376 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041035105315947844, + "loss": 2.6033, + "theoretical_loss": 3.8276013914788862, + "tokens_seen": 618790912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103410230692076, + "loss": 2.9456, + "theoretical_loss": 3.827559322981259, + "tokens_seen": 618856448 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041033099297893685, + "loss": 2.5836, + "theoretical_loss": 3.827517260185637, + "tokens_seen": 618921984 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410320962888666, + "loss": 2.8173, + "theoretical_loss": 3.827475203090642, + "tokens_seen": 618987520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103109327983952, + "loss": 2.898, + "theoretical_loss": 3.827433151694899, + "tokens_seen": 619053056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004103009027081244, + "loss": 2.7016, + "theoretical_loss": 3.827391105997033, + "tokens_seen": 619118592 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102908726178536, + "loss": 2.7453, + "theoretical_loss": 3.8273490659956675, + "tokens_seen": 619184128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041028084252758276, + "loss": 2.6333, + "theoretical_loss": 3.827307031689429, + "tokens_seen": 619249664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 351768, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.970876693725586, + "objective/train/theoretical_loss": 3.8272650030769433, + "objective/train/tokens_used": 639775200, + "theoretical_loss": 3.8272650030769433, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041027081243731194, + "loss": 2.797, + "theoretical_loss": 3.8272650030769433, + "tokens_seen": 619315200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102607823470411, + "loss": 2.6878, + "theoretical_loss": 3.8272229801568365, + "tokens_seen": 619380736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041025075225677035, + "loss": 2.6138, + "theoretical_loss": 3.827180962927735, + "tokens_seen": 619446272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102407221664995, + "loss": 2.7821, + "theoretical_loss": 3.827138951388267, + "tokens_seen": 619511808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102306920762287, + "loss": 2.8132, + "theoretical_loss": 3.8270969455370594, + "tokens_seen": 619577344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041022066198595784, + "loss": 2.7769, + "theoretical_loss": 3.8270549453727414, + "tokens_seen": 619642880 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004102106318956871, + "loss": 2.6755, + "theoretical_loss": 3.8270129508939412, + "tokens_seen": 619708416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041020060180541626, + "loss": 2.593, + "theoretical_loss": 3.8269709620992884, + "tokens_seen": 619773952 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041019057171514544, + "loss": 2.873, + "theoretical_loss": 3.826928978987412, + "tokens_seen": 619839488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101805416248746, + "loss": 2.8805, + "theoretical_loss": 3.8268870015569427, + "tokens_seen": 619905024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101705115346038, + "loss": 2.6737, + "theoretical_loss": 3.826845029806511, + "tokens_seen": 619970560 + }, + { + "epoch": 0.19, + "learning_rate": 0.000410160481444333, + "loss": 2.5944, + "theoretical_loss": 3.826803063734748, + "tokens_seen": 620036096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101504513540622, + "loss": 2.7064, + "theoretical_loss": 3.826761103340286, + "tokens_seen": 620101632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041014042126379135, + "loss": 2.834, + "theoretical_loss": 3.8267191486217556, + "tokens_seen": 620167168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101303911735206, + "loss": 2.5143, + "theoretical_loss": 3.8266771995777904, + "tokens_seen": 620232704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041012036108324976, + "loss": 2.864, + "theoretical_loss": 3.826635256207023, + "tokens_seen": 620298240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041011033099297894, + "loss": 2.5869, + "theoretical_loss": 3.8265933185080874, + "tokens_seen": 620363776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004101003009027081, + "loss": 2.7903, + "theoretical_loss": 3.8265513864796166, + "tokens_seen": 620429312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100902708124373, + "loss": 2.7977, + "theoretical_loss": 3.8265094601202465, + "tokens_seen": 620494848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100802407221665, + "loss": 2.7521, + "theoretical_loss": 3.8264675394286103, + "tokens_seen": 620560384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100702106318957, + "loss": 3.0977, + "theoretical_loss": 3.8264256244033445, + "tokens_seen": 620625920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041006018054162485, + "loss": 2.8243, + "theoretical_loss": 3.826383715043085, + "tokens_seen": 620691456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100501504513541, + "loss": 2.8642, + "theoretical_loss": 3.826341811346468, + "tokens_seen": 620756992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100401203610832, + "loss": 2.6281, + "theoretical_loss": 3.8262999133121296, + "tokens_seen": 620822528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041003009027081245, + "loss": 2.935, + "theoretical_loss": 3.826258020938708, + "tokens_seen": 620888064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 352593, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.924837827682495, + "objective/train/theoretical_loss": 3.82621613422484, + "objective/train/tokens_used": 641413600, + "theoretical_loss": 3.82621613422484, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041002006018054163, + "loss": 2.7235, + "theoretical_loss": 3.82621613422484, + "tokens_seen": 620953600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004100100300902708, + "loss": 2.9794, + "theoretical_loss": 3.8261742531691647, + "tokens_seen": 621019136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00041, + "loss": 2.6264, + "theoretical_loss": 3.8261323777703207, + "tokens_seen": 621084672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040998996990972917, + "loss": 2.7152, + "theoretical_loss": 3.8260905080269465, + "tokens_seen": 621150208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040997993981945835, + "loss": 2.9183, + "theoretical_loss": 3.8260486439376824, + "tokens_seen": 621215744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099699097291876, + "loss": 2.8273, + "theoretical_loss": 3.826006785501168, + "tokens_seen": 621281280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099598796389167, + "loss": 2.8176, + "theoretical_loss": 3.8259649327160448, + "tokens_seen": 621346816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040994984954864595, + "loss": 2.6598, + "theoretical_loss": 3.8259230855809525, + "tokens_seen": 621412352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040993981945837513, + "loss": 2.8564, + "theoretical_loss": 3.825881244094533, + "tokens_seen": 621477888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099297893681043, + "loss": 2.8814, + "theoretical_loss": 3.825839408255429, + "tokens_seen": 621543424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099197592778335, + "loss": 2.97, + "theoretical_loss": 3.825797578062282, + "tokens_seen": 621608960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004099097291875627, + "loss": 2.5886, + "theoretical_loss": 3.825755753513736, + "tokens_seen": 621674496 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098996990972919, + "loss": 2.8166, + "theoretical_loss": 3.8257139346084332, + "tokens_seen": 621740032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098896690070211, + "loss": 2.6562, + "theoretical_loss": 3.8256721213450184, + "tokens_seen": 621805568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098796389167503, + "loss": 2.6982, + "theoretical_loss": 3.8256303137221357, + "tokens_seen": 621871104 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040986960882647945, + "loss": 2.6747, + "theoretical_loss": 3.825588511738429, + "tokens_seen": 621936640 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040985957873620864, + "loss": 2.7269, + "theoretical_loss": 3.8255467153925453, + "tokens_seen": 622002176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098495486459378, + "loss": 2.8566, + "theoretical_loss": 3.8255049246831287, + "tokens_seen": 622067712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040983951855566705, + "loss": 2.934, + "theoretical_loss": 3.825463139608827, + "tokens_seen": 622133248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098294884653962, + "loss": 2.9967, + "theoretical_loss": 3.825421360168285, + "tokens_seen": 622198784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098194583751254, + "loss": 2.5096, + "theoretical_loss": 3.8253795863601505, + "tokens_seen": 622264320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004098094282848546, + "loss": 2.7312, + "theoretical_loss": 3.825337818183072, + "tokens_seen": 622329856 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097993981945838, + "loss": 2.5197, + "theoretical_loss": 3.8252960556356967, + "tokens_seen": 622395392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040978936810431296, + "loss": 2.8355, + "theoretical_loss": 3.8252542987166738, + "tokens_seen": 622460928 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040977933801404214, + "loss": 2.6231, + "theoretical_loss": 3.8252125474246514, + "tokens_seen": 622526464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 353199, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.111541748046875, + "objective/train/theoretical_loss": 3.8251708017582793, + "objective/train/tokens_used": 643052000, + "theoretical_loss": 3.8251708017582793, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097693079237713, + "loss": 2.7705, + "theoretical_loss": 3.8251708017582793, + "tokens_seen": 622592000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040975927783350055, + "loss": 2.8652, + "theoretical_loss": 3.825129061716208, + "tokens_seen": 622657536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097492477432297, + "loss": 2.6621, + "theoretical_loss": 3.825087327297087, + "tokens_seen": 622723072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097392176529589, + "loss": 2.7638, + "theoretical_loss": 3.8250455984995684, + "tokens_seen": 622788608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040972918756268804, + "loss": 2.8562, + "theoretical_loss": 3.825003875322302, + "tokens_seen": 622854144 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004097191574724173, + "loss": 2.762, + "theoretical_loss": 3.8249621577639408, + "tokens_seen": 622919680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040970912738214646, + "loss": 2.8821, + "theoretical_loss": 3.8249204458231363, + "tokens_seen": 622985216 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040969909729187564, + "loss": 2.8805, + "theoretical_loss": 3.8248787394985415, + "tokens_seen": 623050752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096890672016048, + "loss": 2.7553, + "theoretical_loss": 3.82483703878881, + "tokens_seen": 623116288 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409679037111334, + "loss": 2.7981, + "theoretical_loss": 3.824795343692595, + "tokens_seen": 623181824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096690070210632, + "loss": 2.5148, + "theoretical_loss": 3.8247536542085507, + "tokens_seen": 623247360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096589769307924, + "loss": 2.7058, + "theoretical_loss": 3.824711970335332, + "tokens_seen": 623312896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040964894684052155, + "loss": 2.8072, + "theoretical_loss": 3.824670292071593, + "tokens_seen": 623378432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096389167502508, + "loss": 2.7979, + "theoretical_loss": 3.82462861941599, + "tokens_seen": 623443968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040962888665997996, + "loss": 2.6498, + "theoretical_loss": 3.8245869523671794, + "tokens_seen": 623509504 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040961885656970914, + "loss": 2.844, + "theoretical_loss": 3.8245452909238167, + "tokens_seen": 623575040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004096088264794383, + "loss": 2.9147, + "theoretical_loss": 3.824503635084559, + "tokens_seen": 623640576 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095987963891675, + "loss": 2.7143, + "theoretical_loss": 3.8244619848480648, + "tokens_seen": 623706112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095887662988967, + "loss": 2.8396, + "theoretical_loss": 3.82442034021299, + "tokens_seen": 623771648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095787362086259, + "loss": 2.7964, + "theoretical_loss": 3.824378701177994, + "tokens_seen": 623837184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040956870611835505, + "loss": 2.6739, + "theoretical_loss": 3.8243370677417357, + "tokens_seen": 623902720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095586760280843, + "loss": 2.6736, + "theoretical_loss": 3.8242954399028743, + "tokens_seen": 623968256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095486459378134, + "loss": 2.6681, + "theoretical_loss": 3.824253817660069, + "tokens_seen": 624033792 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040953861584754265, + "loss": 2.8982, + "theoretical_loss": 3.82421220101198, + "tokens_seen": 624099328 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040952858575727183, + "loss": 2.914, + "theoretical_loss": 3.8241705899572684, + "tokens_seen": 624164864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 354273, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4973878860473633, + "objective/train/theoretical_loss": 3.8241289844945947, + "objective/train/tokens_used": 644690400, + "theoretical_loss": 3.8241289844945947, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409518555667001, + "loss": 2.747, + "theoretical_loss": 3.8241289844945947, + "tokens_seen": 624230400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004095085255767302, + "loss": 2.6438, + "theoretical_loss": 3.8240873846226204, + "tokens_seen": 624295936 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040949849548645937, + "loss": 2.6574, + "theoretical_loss": 3.8240457903400085, + "tokens_seen": 624361472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040948846539618855, + "loss": 2.902, + "theoretical_loss": 3.82400420164542, + "tokens_seen": 624427008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094784353059178, + "loss": 2.9509, + "theoretical_loss": 3.823962618537519, + "tokens_seen": 624492544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094684052156469, + "loss": 2.9318, + "theoretical_loss": 3.8239210410149678, + "tokens_seen": 624558080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040945837512537615, + "loss": 2.6767, + "theoretical_loss": 3.823879469076431, + "tokens_seen": 624623616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040944834503510533, + "loss": 2.8474, + "theoretical_loss": 3.8238379027205727, + "tokens_seen": 624689152 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094383149448345, + "loss": 2.7381, + "theoretical_loss": 3.823796341946058, + "tokens_seen": 624754688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094282848545637, + "loss": 2.8622, + "theoretical_loss": 3.823754786751551, + "tokens_seen": 624820224 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004094182547642929, + "loss": 2.8514, + "theoretical_loss": 3.8237132371357188, + "tokens_seen": 624885760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040940822467402206, + "loss": 2.8813, + "theoretical_loss": 3.823671693097226, + "tokens_seen": 624951296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093981945837513, + "loss": 2.614, + "theoretical_loss": 3.8236301546347407, + "tokens_seen": 625016832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093881644934804, + "loss": 2.7085, + "theoretical_loss": 3.8235886217469286, + "tokens_seen": 625082368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040937813440320965, + "loss": 2.7141, + "theoretical_loss": 3.8235470944324588, + "tokens_seen": 625147904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093681043129388, + "loss": 2.7833, + "theoretical_loss": 3.823505572689997, + "tokens_seen": 625213440 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409358074222668, + "loss": 2.8488, + "theoretical_loss": 3.8234640565182136, + "tokens_seen": 625278976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093480441323972, + "loss": 2.8908, + "theoretical_loss": 3.8234225459157773, + "tokens_seen": 625344512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093380140421264, + "loss": 2.879, + "theoretical_loss": 3.823381040881356, + "tokens_seen": 625410048 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040932798395185556, + "loss": 2.656, + "theoretical_loss": 3.8233395414136204, + "tokens_seen": 625475584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093179538615848, + "loss": 2.5753, + "theoretical_loss": 3.8232980475112415, + "tokens_seen": 625541120 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004093079237713139, + "loss": 2.5492, + "theoretical_loss": 3.8232565591728886, + "tokens_seen": 625606656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040929789368104316, + "loss": 2.7042, + "theoretical_loss": 3.8232150763972337, + "tokens_seen": 625672192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092878635907723, + "loss": 2.7082, + "theoretical_loss": 3.8231735991829483, + "tokens_seen": 625737728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092778335005015, + "loss": 2.8102, + "theoretical_loss": 3.8231321275287042, + "tokens_seen": 625803264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 354855, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.032952308654785, + "objective/train/theoretical_loss": 3.8230906614331737, + "objective/train/tokens_used": 646328800, + "theoretical_loss": 3.8230906614331737, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092678034102307, + "loss": 2.8881, + "theoretical_loss": 3.8230906614331737, + "tokens_seen": 625868800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092577733199599, + "loss": 2.8983, + "theoretical_loss": 3.823049200895031, + "tokens_seen": 625934336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040924774322968906, + "loss": 2.857, + "theoretical_loss": 3.823007745912948, + "tokens_seen": 625999872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040923771313941824, + "loss": 3.0152, + "theoretical_loss": 3.8229662964855997, + "tokens_seen": 626065408 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092276830491474, + "loss": 2.9206, + "theoretical_loss": 3.82292485261166, + "tokens_seen": 626130944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040921765295887666, + "loss": 2.6842, + "theoretical_loss": 3.822883414289804, + "tokens_seen": 626196480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004092076228686058, + "loss": 2.9224, + "theoretical_loss": 3.8228419815187062, + "tokens_seen": 626262016 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409197592778335, + "loss": 2.6684, + "theoretical_loss": 3.822800554297043, + "tokens_seen": 626327552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040918756268806415, + "loss": 2.9722, + "theoretical_loss": 3.8227591326234904, + "tokens_seen": 626393088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091775325977934, + "loss": 2.6933, + "theoretical_loss": 3.822717716496725, + "tokens_seen": 626458624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040916750250752257, + "loss": 2.9309, + "theoretical_loss": 3.822676305915424, + "tokens_seen": 626524160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040915747241725175, + "loss": 2.9623, + "theoretical_loss": 3.822634900878265, + "tokens_seen": 626589696 + }, + { + "epoch": 0.19, + "learning_rate": 0.000409147442326981, + "loss": 2.7984, + "theoretical_loss": 3.8225935013839254, + "tokens_seen": 626655232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040913741223671016, + "loss": 2.7617, + "theoretical_loss": 3.8225521074310844, + "tokens_seen": 626720768 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040912738214643934, + "loss": 2.9621, + "theoretical_loss": 3.82251071901842, + "tokens_seen": 626786304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091173520561685, + "loss": 2.7483, + "theoretical_loss": 3.8224693361446134, + "tokens_seen": 626851840 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004091073219658977, + "loss": 2.4055, + "theoretical_loss": 3.822427958808342, + "tokens_seen": 626917376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090972918756269, + "loss": 2.9295, + "theoretical_loss": 3.822386587008287, + "tokens_seen": 626982912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090872617853561, + "loss": 2.787, + "theoretical_loss": 3.82234522074313, + "tokens_seen": 627048448 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040907723169508525, + "loss": 2.6379, + "theoretical_loss": 3.8223038600115506, + "tokens_seen": 627113984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090672016048145, + "loss": 2.8665, + "theoretical_loss": 3.8222625048122314, + "tokens_seen": 627179520 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090571715145436, + "loss": 2.844, + "theoretical_loss": 3.8222211551438545, + "tokens_seen": 627245056 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040904714142427285, + "loss": 2.5677, + "theoretical_loss": 3.8221798110051024, + "tokens_seen": 627310592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040903711133400203, + "loss": 2.8402, + "theoretical_loss": 3.8221384723946574, + "tokens_seen": 627376128 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090270812437312, + "loss": 2.7906, + "theoretical_loss": 3.8220971393112033, + "tokens_seen": 627441664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 356151, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0492939949035645, + "objective/train/theoretical_loss": 3.8220558117534242, + "objective/train/tokens_used": 647967200, + "theoretical_loss": 3.8220558117534242, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004090170511534604, + "loss": 2.7642, + "theoretical_loss": 3.8220558117534242, + "tokens_seen": 627507200 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040900702106318957, + "loss": 2.7001, + "theoretical_loss": 3.8220144897200035, + "tokens_seen": 627572736 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040899699097291875, + "loss": 3.154, + "theoretical_loss": 3.821973173209628, + "tokens_seen": 627638272 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408986960882648, + "loss": 2.9608, + "theoretical_loss": 3.82193186222098, + "tokens_seen": 627703808 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089769307923771, + "loss": 2.7454, + "theoretical_loss": 3.821890556752748, + "tokens_seen": 627769344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040896690070210635, + "loss": 2.8077, + "theoretical_loss": 3.821849256803617, + "tokens_seen": 627834880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040895687061183553, + "loss": 2.6292, + "theoretical_loss": 3.8218079623722723, + "tokens_seen": 627900416 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089468405215647, + "loss": 2.6008, + "theoretical_loss": 3.821766673457403, + "tokens_seen": 627965952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089368104312939, + "loss": 2.8147, + "theoretical_loss": 3.8217253900576953, + "tokens_seen": 628031488 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089267803410231, + "loss": 2.9192, + "theoretical_loss": 3.821684112171837, + "tokens_seen": 628097024 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040891675025075226, + "loss": 3.0209, + "theoretical_loss": 3.8216428397985176, + "tokens_seen": 628162560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004089067201604815, + "loss": 2.6749, + "theoretical_loss": 3.8216015729364248, + "tokens_seen": 628228096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088966900702106, + "loss": 2.847, + "theoretical_loss": 3.821560311584248, + "tokens_seen": 628293632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040888665997993985, + "loss": 2.6515, + "theoretical_loss": 3.8215190557406773, + "tokens_seen": 628359168 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408876629889669, + "loss": 2.6926, + "theoretical_loss": 3.821477805404403, + "tokens_seen": 628424704 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088665997993982, + "loss": 2.9088, + "theoretical_loss": 3.8214365605741154, + "tokens_seen": 628490240 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088565697091274, + "loss": 2.7691, + "theoretical_loss": 3.8213953212485055, + "tokens_seen": 628555776 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088465396188566, + "loss": 2.8476, + "theoretical_loss": 3.821354087426265, + "tokens_seen": 628621312 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040883650952858576, + "loss": 2.6684, + "theoretical_loss": 3.821312859106085, + "tokens_seen": 628686848 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408826479438315, + "loss": 2.7181, + "theoretical_loss": 3.8212716362866597, + "tokens_seen": 628752384 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004088164493480441, + "loss": 2.5384, + "theoretical_loss": 3.8212304189666804, + "tokens_seen": 628817920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040880641925777336, + "loss": 2.6511, + "theoretical_loss": 3.8211892071448403, + "tokens_seen": 628883456 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087963891675025, + "loss": 2.672, + "theoretical_loss": 3.8211480008198344, + "tokens_seen": 628948992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087863590772317, + "loss": 2.6401, + "theoretical_loss": 3.821106799990356, + "tokens_seen": 629014528 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087763289869609, + "loss": 2.6822, + "theoretical_loss": 3.8210656046550997, + "tokens_seen": 629080064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 356825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.88836932182312, + "objective/train/theoretical_loss": 3.821024414812761, + "objective/train/tokens_used": 649605600, + "theoretical_loss": 3.821024414812761, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087662988966901, + "loss": 2.7571, + "theoretical_loss": 3.821024414812761, + "tokens_seen": 629145600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040875626880641926, + "loss": 2.8849, + "theoretical_loss": 3.8209832304620353, + "tokens_seen": 629211136 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040874623871614844, + "loss": 2.8463, + "theoretical_loss": 3.8209420516016177, + "tokens_seen": 629276672 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087362086258776, + "loss": 2.5645, + "theoretical_loss": 3.820900878230206, + "tokens_seen": 629342208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040872617853560686, + "loss": 2.9021, + "theoretical_loss": 3.8208597103464963, + "tokens_seen": 629407744 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408716148445336, + "loss": 2.8184, + "theoretical_loss": 3.820818547949186, + "tokens_seen": 629473280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004087061183550652, + "loss": 2.9188, + "theoretical_loss": 3.820777391036973, + "tokens_seen": 629538816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040869608826479435, + "loss": 3.0061, + "theoretical_loss": 3.8207362396085554, + "tokens_seen": 629604352 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086860581745236, + "loss": 2.9378, + "theoretical_loss": 3.820695093662631, + "tokens_seen": 629669888 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040867602808425277, + "loss": 2.6738, + "theoretical_loss": 3.8206539531979002, + "tokens_seen": 629735424 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040866599799398195, + "loss": 2.6967, + "theoretical_loss": 3.8206128182130623, + "tokens_seen": 629800960 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040865596790371113, + "loss": 2.6859, + "theoretical_loss": 3.8205716887068166, + "tokens_seen": 629866496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040864593781344036, + "loss": 2.8031, + "theoretical_loss": 3.8205305646778633, + "tokens_seen": 629932032 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086359077231695, + "loss": 2.8133, + "theoretical_loss": 3.820489446124905, + "tokens_seen": 629997568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086258776328987, + "loss": 2.8721, + "theoretical_loss": 3.8204483330466408, + "tokens_seen": 630063104 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040861584754262785, + "loss": 3.052, + "theoretical_loss": 3.820407225441774, + "tokens_seen": 630128640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004086058174523571, + "loss": 3.0175, + "theoretical_loss": 3.820366123309006, + "tokens_seen": 630194176 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040859578736208627, + "loss": 2.6538, + "theoretical_loss": 3.82032502664704, + "tokens_seen": 630259712 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040858575727181545, + "loss": 2.8017, + "theoretical_loss": 3.8202839354545786, + "tokens_seen": 630325248 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040857572718154463, + "loss": 2.8723, + "theoretical_loss": 3.8202428497303247, + "tokens_seen": 630390784 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085656970912738, + "loss": 2.8212, + "theoretical_loss": 3.820201769472984, + "tokens_seen": 630456320 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408555667001003, + "loss": 2.9297, + "theoretical_loss": 3.8201606946812596, + "tokens_seen": 630521856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040854563691073223, + "loss": 2.9594, + "theoretical_loss": 3.820119625353856, + "tokens_seen": 630587392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040853560682046136, + "loss": 2.7654, + "theoretical_loss": 3.8200785614894794, + "tokens_seen": 630652928 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085255767301906, + "loss": 2.7078, + "theoretical_loss": 3.820037503086836, + "tokens_seen": 630718464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 357395, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.229945182800293, + "objective/train/theoretical_loss": 3.81999645014463, + "objective/train/tokens_used": 651244000, + "theoretical_loss": 3.81999645014463, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004085155466399197, + "loss": 2.9155, + "theoretical_loss": 3.81999645014463, + "tokens_seen": 630784000 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040850551654964895, + "loss": 2.7578, + "theoretical_loss": 3.81995540266157, + "tokens_seen": 630849536 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040849548645937814, + "loss": 2.7972, + "theoretical_loss": 3.819914360636362, + "tokens_seen": 630915072 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084854563691073, + "loss": 2.9271, + "theoretical_loss": 3.8198733240677134, + "tokens_seen": 630980608 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084754262788365, + "loss": 2.8088, + "theoretical_loss": 3.8198322929543327, + "tokens_seen": 631046144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040846539618856573, + "loss": 2.8873, + "theoretical_loss": 3.819791267294928, + "tokens_seen": 631111680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040845536609829486, + "loss": 2.764, + "theoretical_loss": 3.8197502470882077, + "tokens_seen": 631177216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084453360080241, + "loss": 2.6695, + "theoretical_loss": 3.8197092323328814, + "tokens_seen": 631242752 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084353059177532, + "loss": 3.0271, + "theoretical_loss": 3.8196682230276595, + "tokens_seen": 631308288 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040842527582748246, + "loss": 2.8736, + "theoretical_loss": 3.8196272191712506, + "tokens_seen": 631373824 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040841524573721164, + "loss": 2.8502, + "theoretical_loss": 3.8195862207623668, + "tokens_seen": 631439360 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004084052156469408, + "loss": 2.6863, + "theoretical_loss": 3.8195452277997184, + "tokens_seen": 631504896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040839518555667005, + "loss": 2.6358, + "theoretical_loss": 3.819504240282016, + "tokens_seen": 631570432 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083851554663992, + "loss": 2.8913, + "theoretical_loss": 3.819463258207973, + "tokens_seen": 631635968 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083751253761284, + "loss": 2.8457, + "theoretical_loss": 3.8194222815763004, + "tokens_seen": 631701504 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083650952858576, + "loss": 3.0551, + "theoretical_loss": 3.8193813103857126, + "tokens_seen": 631767040 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083550651955868, + "loss": 2.7654, + "theoretical_loss": 3.8193403446349206, + "tokens_seen": 631832576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040834503510531596, + "loss": 2.8818, + "theoretical_loss": 3.8192993843226395, + "tokens_seen": 631898112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083350050150452, + "loss": 2.9639, + "theoretical_loss": 3.8192584294475833, + "tokens_seen": 631963648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083249749247743, + "loss": 2.812, + "theoretical_loss": 3.8192174800084664, + "tokens_seen": 632029184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040831494483450356, + "loss": 2.7267, + "theoretical_loss": 3.819176536004003, + "tokens_seen": 632094720 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004083049147442327, + "loss": 2.8316, + "theoretical_loss": 3.8191355974329095, + "tokens_seen": 632160256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082948846539619, + "loss": 2.7239, + "theoretical_loss": 3.8190946642939014, + "tokens_seen": 632225792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082848545636911, + "loss": 2.6075, + "theoretical_loss": 3.819053736585694, + "tokens_seen": 632291328 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082748244734203, + "loss": 2.7152, + "theoretical_loss": 3.8190128143070057, + "tokens_seen": 632356864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 358555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.62202525138855, + "objective/train/theoretical_loss": 3.818971897456552, + "objective/train/tokens_used": 652882400, + "theoretical_loss": 3.818971897456552, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040826479438314946, + "loss": 2.9621, + "theoretical_loss": 3.818971897456552, + "tokens_seen": 632422400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040825476429287864, + "loss": 2.7414, + "theoretical_loss": 3.818930986033052, + "tokens_seen": 632487936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082447342026078, + "loss": 2.7112, + "theoretical_loss": 3.8188900800352226, + "tokens_seen": 632553472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040823470411233706, + "loss": 2.9688, + "theoretical_loss": 3.8188491794617825, + "tokens_seen": 632619008 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082246740220662, + "loss": 2.9359, + "theoretical_loss": 3.81880828431145, + "tokens_seen": 632684544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004082146439317954, + "loss": 2.7537, + "theoretical_loss": 3.8187673945829457, + "tokens_seen": 632750080 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040820461384152455, + "loss": 2.583, + "theoretical_loss": 3.8187265102749883, + "tokens_seen": 632815616 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081945837512538, + "loss": 2.8925, + "theoretical_loss": 3.8186856313862982, + "tokens_seen": 632881152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040818455366098297, + "loss": 2.487, + "theoretical_loss": 3.818644757915597, + "tokens_seen": 632946688 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040817452357071215, + "loss": 2.7202, + "theoretical_loss": 3.8186038898616035, + "tokens_seen": 633012224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040816449348044133, + "loss": 2.8124, + "theoretical_loss": 3.8185630272230413, + "tokens_seen": 633077760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040815446339017056, + "loss": 2.66, + "theoretical_loss": 3.8185221699986314, + "tokens_seen": 633143296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081444332998997, + "loss": 2.8485, + "theoretical_loss": 3.818481318187096, + "tokens_seen": 633208832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081344032096289, + "loss": 2.8472, + "theoretical_loss": 3.8184404717871585, + "tokens_seen": 633274368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040812437311935805, + "loss": 2.766, + "theoretical_loss": 3.8183996307975416, + "tokens_seen": 633339904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004081143430290873, + "loss": 2.5663, + "theoretical_loss": 3.8183587952169686, + "tokens_seen": 633405440 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040810431293881647, + "loss": 2.7984, + "theoretical_loss": 3.818317965044165, + "tokens_seen": 633470976 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040809428284854565, + "loss": 2.9312, + "theoretical_loss": 3.818277140277854, + "tokens_seen": 633536512 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040808425275827483, + "loss": 2.6212, + "theoretical_loss": 3.8182363209167605, + "tokens_seen": 633602048 + }, + { + "epoch": 0.19, + "learning_rate": 0.000408074222668004, + "loss": 2.8187, + "theoretical_loss": 3.8181955069596105, + "tokens_seen": 633667584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080641925777332, + "loss": 2.7484, + "theoretical_loss": 3.81815469840513, + "tokens_seen": 633733120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040805416248746243, + "loss": 2.8055, + "theoretical_loss": 3.8181138952520444, + "tokens_seen": 633798656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040804413239719156, + "loss": 2.805, + "theoretical_loss": 3.8180730974990813, + "tokens_seen": 633864192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080341023069208, + "loss": 2.8455, + "theoretical_loss": 3.8180323051449667, + "tokens_seen": 633929728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004080240722166499, + "loss": 2.824, + "theoretical_loss": 3.817991518188429, + "tokens_seen": 633995264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 359090, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7171945571899414, + "objective/train/theoretical_loss": 3.8179507366281964, + "objective/train/tokens_used": 654520800, + "theoretical_loss": 3.8179507366281964, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040801404212637915, + "loss": 2.8011, + "theoretical_loss": 3.8179507366281964, + "tokens_seen": 634060800 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040800401203610834, + "loss": 2.6352, + "theoretical_loss": 3.817909960462996, + "tokens_seen": 634126336 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079939819458375, + "loss": 2.7008, + "theoretical_loss": 3.817869189691558, + "tokens_seen": 634191872 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079839518555667, + "loss": 2.4876, + "theoretical_loss": 3.8178284243126113, + "tokens_seen": 634257408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040797392176529593, + "loss": 2.5169, + "theoretical_loss": 3.8177876643248845, + "tokens_seen": 634322944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040796389167502506, + "loss": 2.5934, + "theoretical_loss": 3.8177469097271093, + "tokens_seen": 634388480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079538615847543, + "loss": 2.5588, + "theoretical_loss": 3.817706160518015, + "tokens_seen": 634454016 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079438314944834, + "loss": 2.6794, + "theoretical_loss": 3.8176654166963337, + "tokens_seen": 634519552 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040793380140421266, + "loss": 2.5933, + "theoretical_loss": 3.8176246782607963, + "tokens_seen": 634585088 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040792377131394184, + "loss": 2.6201, + "theoretical_loss": 3.817583945210134, + "tokens_seen": 634650624 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407913741223671, + "loss": 2.9393, + "theoretical_loss": 3.8175432175430797, + "tokens_seen": 634716160 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004079037111334002, + "loss": 2.8609, + "theoretical_loss": 3.817502495258366, + "tokens_seen": 634781696 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078936810431294, + "loss": 2.7891, + "theoretical_loss": 3.817461778354727, + "tokens_seen": 634847232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040788365095285856, + "loss": 2.9239, + "theoretical_loss": 3.8174210668308945, + "tokens_seen": 634912768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078736208625878, + "loss": 2.7264, + "theoretical_loss": 3.8173803606856036, + "tokens_seen": 634978304 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078635907723169, + "loss": 2.7941, + "theoretical_loss": 3.8173396599175886, + "tokens_seen": 635043840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040785356068204616, + "loss": 2.8307, + "theoretical_loss": 3.817298964525584, + "tokens_seen": 635109376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078435305917753, + "loss": 2.8384, + "theoretical_loss": 3.817258274508325, + "tokens_seen": 635174912 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078335005015045, + "loss": 2.7362, + "theoretical_loss": 3.8172175898645477, + "tokens_seen": 635240448 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078234704112337, + "loss": 2.5734, + "theoretical_loss": 3.817176910592988, + "tokens_seen": 635305984 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004078134403209629, + "loss": 2.732, + "theoretical_loss": 3.8171362366923827, + "tokens_seen": 635371520 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040780341023069207, + "loss": 2.6684, + "theoretical_loss": 3.817095568161468, + "tokens_seen": 635437056 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077933801404213, + "loss": 2.8199, + "theoretical_loss": 3.817054904998983, + "tokens_seen": 635502592 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040778335005015043, + "loss": 2.601, + "theoretical_loss": 3.817014247203664, + "tokens_seen": 635568128 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040777331995987966, + "loss": 2.6932, + "theoretical_loss": 3.8169735947742494, + "tokens_seen": 635633664 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 360366, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.813488483428955, + "objective/train/theoretical_loss": 3.8169329477094784, + "objective/train/tokens_used": 656159200, + "theoretical_loss": 3.8169329477094784, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077632898696088, + "loss": 2.7224, + "theoretical_loss": 3.8169329477094784, + "tokens_seen": 635699200 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407753259779338, + "loss": 2.7715, + "theoretical_loss": 3.81689230600809, + "tokens_seen": 635764736 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077432296890672, + "loss": 2.6614, + "theoretical_loss": 3.8168516696688233, + "tokens_seen": 635830272 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004077331995987964, + "loss": 2.7141, + "theoretical_loss": 3.8168110386904193, + "tokens_seen": 635895808 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040772316950852557, + "loss": 2.8513, + "theoretical_loss": 3.8167704130716174, + "tokens_seen": 635961344 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040771313941825475, + "loss": 2.8096, + "theoretical_loss": 3.8167297928111585, + "tokens_seen": 636026880 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040770310932798393, + "loss": 2.5866, + "theoretical_loss": 3.8166891779077847, + "tokens_seen": 636092416 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040769307923771317, + "loss": 3.1336, + "theoretical_loss": 3.816648568360236, + "tokens_seen": 636157952 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076830491474423, + "loss": 2.6937, + "theoretical_loss": 3.8166079641672566, + "tokens_seen": 636223488 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040767301905717153, + "loss": 2.7533, + "theoretical_loss": 3.8165673653275873, + "tokens_seen": 636289024 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076629889669007, + "loss": 2.8758, + "theoretical_loss": 3.816526771839972, + "tokens_seen": 636354560 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076529588766299, + "loss": 2.6713, + "theoretical_loss": 3.816486183703154, + "tokens_seen": 636420096 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076429287863591, + "loss": 3.0028, + "theoretical_loss": 3.8164456009158765, + "tokens_seen": 636485632 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040763289869608825, + "loss": 2.6476, + "theoretical_loss": 3.816405023476884, + "tokens_seen": 636551168 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004076228686058175, + "loss": 3.0361, + "theoretical_loss": 3.8163644513849215, + "tokens_seen": 636616704 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040761283851554667, + "loss": 2.4477, + "theoretical_loss": 3.816323884638734, + "tokens_seen": 636682240 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040760280842527585, + "loss": 2.7836, + "theoretical_loss": 3.8162833232370668, + "tokens_seen": 636747776 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040759277833500503, + "loss": 2.7696, + "theoretical_loss": 3.8162427671786654, + "tokens_seen": 636813312 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075827482447342, + "loss": 2.6237, + "theoretical_loss": 3.8162022164622766, + "tokens_seen": 636878848 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075727181544634, + "loss": 2.7036, + "theoretical_loss": 3.8161616710866473, + "tokens_seen": 636944384 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040756268806419263, + "loss": 2.7311, + "theoretical_loss": 3.8161211310505245, + "tokens_seen": 637009920 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040755265797392176, + "loss": 2.8929, + "theoretical_loss": 3.8160805963526556, + "tokens_seen": 637075456 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407542627883651, + "loss": 2.9721, + "theoretical_loss": 3.816040066991789, + "tokens_seen": 637140992 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075325977933801, + "loss": 2.9736, + "theoretical_loss": 3.815999542966673, + "tokens_seen": 637206528 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040752256770310935, + "loss": 2.5887, + "theoretical_loss": 3.8159590242760566, + "tokens_seen": 637272064 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 360972, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.695178270339966, + "objective/train/theoretical_loss": 3.8159185109186886, + "objective/train/tokens_used": 657797600, + "theoretical_loss": 3.8159185109186886, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040751253761283854, + "loss": 2.7188, + "theoretical_loss": 3.8159185109186886, + "tokens_seen": 637337600 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004075025075225677, + "loss": 2.8754, + "theoretical_loss": 3.815878002893319, + "tokens_seen": 637403136 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074924774322969, + "loss": 2.8012, + "theoretical_loss": 3.8158375001986986, + "tokens_seen": 637468672 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040748244734202613, + "loss": 2.7555, + "theoretical_loss": 3.815797002833577, + "tokens_seen": 637534208 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040747241725175526, + "loss": 2.6793, + "theoretical_loss": 3.815756510796705, + "tokens_seen": 637599744 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074623871614845, + "loss": 2.8207, + "theoretical_loss": 3.8157160240868353, + "tokens_seen": 637665280 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074523570712136, + "loss": 2.9315, + "theoretical_loss": 3.815675542702719, + "tokens_seen": 637730816 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040744232698094286, + "loss": 2.6283, + "theoretical_loss": 3.8156350666431074, + "tokens_seen": 637796352 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040743229689067204, + "loss": 2.8794, + "theoretical_loss": 3.815594595906755, + "tokens_seen": 637861888 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074222668004012, + "loss": 2.7645, + "theoretical_loss": 3.815554130492414, + "tokens_seen": 637927424 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074122367101304, + "loss": 2.61, + "theoretical_loss": 3.815513670398837, + "tokens_seen": 637992960 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004074022066198596, + "loss": 2.7538, + "theoretical_loss": 3.8154732156247793, + "tokens_seen": 638058496 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040739217652958876, + "loss": 2.7017, + "theoretical_loss": 3.8154327661689944, + "tokens_seen": 638124032 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407382146439318, + "loss": 2.7127, + "theoretical_loss": 3.815392322030238, + "tokens_seen": 638189568 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073721163490471, + "loss": 2.6691, + "theoretical_loss": 3.8153518832072644, + "tokens_seen": 638255104 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040736208625877636, + "loss": 2.5609, + "theoretical_loss": 3.8153114496988296, + "tokens_seen": 638320640 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073520561685055, + "loss": 2.9655, + "theoretical_loss": 3.8152710215036896, + "tokens_seen": 638386176 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073420260782347, + "loss": 2.825, + "theoretical_loss": 3.8152305986206, + "tokens_seen": 638451712 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073319959879639, + "loss": 2.7397, + "theoretical_loss": 3.815190181048319, + "tokens_seen": 638517248 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073219658976931, + "loss": 2.8857, + "theoretical_loss": 3.815149768785603, + "tokens_seen": 638582784 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040731193580742227, + "loss": 2.8068, + "theoretical_loss": 3.81510936183121, + "tokens_seen": 638648320 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004073019057171515, + "loss": 2.6231, + "theoretical_loss": 3.8150689601838983, + "tokens_seen": 638713856 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040729187562688063, + "loss": 2.9175, + "theoretical_loss": 3.815028563842426, + "tokens_seen": 638779392 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040728184553660986, + "loss": 3.0054, + "theoretical_loss": 3.8149881728055526, + "tokens_seen": 638844928 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407271815446339, + "loss": 2.8376, + "theoretical_loss": 3.814947787072037, + "tokens_seen": 638910464 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 361973, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6586904525756836, + "objective/train/theoretical_loss": 3.814907406640639, + "objective/train/tokens_used": 659436000, + "theoretical_loss": 3.814907406640639, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072617853560682, + "loss": 2.7195, + "theoretical_loss": 3.814907406640639, + "tokens_seen": 638976000 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072517552657974, + "loss": 2.8213, + "theoretical_loss": 3.8148670315101185, + "tokens_seen": 639041536 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004072417251755266, + "loss": 2.882, + "theoretical_loss": 3.8148266616792377, + "tokens_seen": 639107072 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040723169508525577, + "loss": 2.8047, + "theoretical_loss": 3.814786297146756, + "tokens_seen": 639172608 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040722166499498495, + "loss": 2.7604, + "theoretical_loss": 3.8147459379114346, + "tokens_seen": 639238144 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040721163490471413, + "loss": 2.8299, + "theoretical_loss": 3.8147055839720365, + "tokens_seen": 639303680 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040720160481444337, + "loss": 2.827, + "theoretical_loss": 3.8146652353273236, + "tokens_seen": 639369216 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071915747241725, + "loss": 3.0141, + "theoretical_loss": 3.8146248919760586, + "tokens_seen": 639434752 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040718154463390173, + "loss": 2.6655, + "theoretical_loss": 3.8145845539170047, + "tokens_seen": 639500288 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071715145436309, + "loss": 2.9756, + "theoretical_loss": 3.8145442211489256, + "tokens_seen": 639565824 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004071614844533601, + "loss": 2.6459, + "theoretical_loss": 3.8145038936705844, + "tokens_seen": 639631360 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040715145436308927, + "loss": 2.7277, + "theoretical_loss": 3.814463571480746, + "tokens_seen": 639696896 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040714142427281845, + "loss": 2.6967, + "theoretical_loss": 3.814423254578176, + "tokens_seen": 639762432 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040713139418254763, + "loss": 2.9031, + "theoretical_loss": 3.8143829429616383, + "tokens_seen": 639827968 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040712136409227687, + "loss": 2.8567, + "theoretical_loss": 3.814342636629899, + "tokens_seen": 639893504 + }, + { + "epoch": 0.19, + "learning_rate": 0.000407111334002006, + "loss": 2.6368, + "theoretical_loss": 3.814302335581724, + "tokens_seen": 639959040 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040710130391173523, + "loss": 2.7911, + "theoretical_loss": 3.81426203981588, + "tokens_seen": 640024576 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040709127382146436, + "loss": 2.9786, + "theoretical_loss": 3.814221749331134, + "tokens_seen": 640090112 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070812437311936, + "loss": 2.7101, + "theoretical_loss": 3.8141814641262526, + "tokens_seen": 640155648 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070712136409228, + "loss": 2.7521, + "theoretical_loss": 3.814141184200004, + "tokens_seen": 640221184 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040706118355065196, + "loss": 2.8041, + "theoretical_loss": 3.814100909551156, + "tokens_seen": 640286720 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040705115346038114, + "loss": 2.8164, + "theoretical_loss": 3.814060640178478, + "tokens_seen": 640352256 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070411233701103, + "loss": 2.9017, + "theoretical_loss": 3.8140203760807374, + "tokens_seen": 640417792 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070310932798395, + "loss": 2.7351, + "theoretical_loss": 3.8139801172567047, + "tokens_seen": 640483328 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040702106318956874, + "loss": 2.7543, + "theoretical_loss": 3.8139398637051496, + "tokens_seen": 640548864 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 362406, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6734325885772705, + "objective/train/theoretical_loss": 3.813899615424841, + "objective/train/tokens_used": 661074400, + "theoretical_loss": 3.813899615424841, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040701103309929786, + "loss": 2.9096, + "theoretical_loss": 3.813899615424841, + "tokens_seen": 640614400 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004070010030090271, + "loss": 2.8807, + "theoretical_loss": 3.8138593724145515, + "tokens_seen": 640679936 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069909729187563, + "loss": 2.6374, + "theoretical_loss": 3.81381913467305, + "tokens_seen": 640745472 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040698094282848546, + "loss": 2.6867, + "theoretical_loss": 3.8137789021991093, + "tokens_seen": 640811008 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040697091273821464, + "loss": 2.7622, + "theoretical_loss": 3.813738674991501, + "tokens_seen": 640876544 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069608826479438, + "loss": 2.7671, + "theoretical_loss": 3.813698453048997, + "tokens_seen": 640942080 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406950852557673, + "loss": 2.7282, + "theoretical_loss": 3.81365823637037, + "tokens_seen": 641007616 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040694082246740224, + "loss": 2.8956, + "theoretical_loss": 3.813618024954393, + "tokens_seen": 641073152 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040693079237713137, + "loss": 2.8595, + "theoretical_loss": 3.81357781879984, + "tokens_seen": 641138688 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004069207622868606, + "loss": 2.797, + "theoretical_loss": 3.8135376179054843, + "tokens_seen": 641204224 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040691073219658973, + "loss": 2.9285, + "theoretical_loss": 3.8134974222700997, + "tokens_seen": 641269760 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040690070210631896, + "loss": 2.9507, + "theoretical_loss": 3.813457231892462, + "tokens_seen": 641335296 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068906720160482, + "loss": 2.9627, + "theoretical_loss": 3.8134170467713453, + "tokens_seen": 641400832 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068806419257773, + "loss": 2.7184, + "theoretical_loss": 3.813376866905526, + "tokens_seen": 641466368 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040687061183550656, + "loss": 2.9675, + "theoretical_loss": 3.81333669229378, + "tokens_seen": 641531904 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068605817452357, + "loss": 2.7346, + "theoretical_loss": 3.813296522934883, + "tokens_seen": 641597440 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068505516549649, + "loss": 2.7329, + "theoretical_loss": 3.813256358827611, + "tokens_seen": 641662976 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068405215646941, + "loss": 2.6317, + "theoretical_loss": 3.8132161999707437, + "tokens_seen": 641728512 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068304914744233, + "loss": 2.8005, + "theoretical_loss": 3.813176046363056, + "tokens_seen": 641794048 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040682046138415247, + "loss": 2.8122, + "theoretical_loss": 3.8131358980033276, + "tokens_seen": 641859584 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004068104312938817, + "loss": 2.8557, + "theoretical_loss": 3.8130957548903357, + "tokens_seen": 641925120 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040680040120361083, + "loss": 2.8691, + "theoretical_loss": 3.81305561702286, + "tokens_seen": 641990656 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040679037111334006, + "loss": 2.6403, + "theoretical_loss": 3.8130154843996795, + "tokens_seen": 642056192 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067803410230692, + "loss": 2.8516, + "theoretical_loss": 3.812975357019573, + "tokens_seen": 642121728 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067703109327984, + "loss": 3.0307, + "theoretical_loss": 3.812935234881322, + "tokens_seen": 642187264 + }, + { + "epoch": 0.19, + "objective/train/docs_used": 363472, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8334643840789795, + "objective/train/theoretical_loss": 3.8128951179837056, + "objective/train/tokens_used": 662712800, + "theoretical_loss": 3.8128951179837056, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067602808425276, + "loss": 2.7628, + "theoretical_loss": 3.8128951179837056, + "tokens_seen": 642252800 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067502507522568, + "loss": 2.6521, + "theoretical_loss": 3.812855006325505, + "tokens_seen": 642318336 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040674022066198597, + "loss": 2.7719, + "theoretical_loss": 3.8128148999055016, + "tokens_seen": 642383872 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040673019057171515, + "loss": 2.9862, + "theoretical_loss": 3.8127747987224767, + "tokens_seen": 642449408 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040672016048144433, + "loss": 2.7238, + "theoretical_loss": 3.812734702775213, + "tokens_seen": 642514944 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040671013039117357, + "loss": 2.7985, + "theoretical_loss": 3.8126946120624927, + "tokens_seen": 642580480 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004067001003009027, + "loss": 2.9992, + "theoretical_loss": 3.8126545265830982, + "tokens_seen": 642646016 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040669007021063193, + "loss": 2.7378, + "theoretical_loss": 3.8126144463358136, + "tokens_seen": 642711552 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066800401203611, + "loss": 2.7071, + "theoretical_loss": 3.812574371319422, + "tokens_seen": 642777088 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066700100300903, + "loss": 2.7973, + "theoretical_loss": 3.812534301532707, + "tokens_seen": 642842624 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040665997993981947, + "loss": 2.8874, + "theoretical_loss": 3.8124942369744543, + "tokens_seen": 642908160 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040664994984954865, + "loss": 2.7893, + "theoretical_loss": 3.812454177643448, + "tokens_seen": 642973696 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040663991975927783, + "loss": 2.7979, + "theoretical_loss": 3.812414123538474, + "tokens_seen": 643039232 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040662988966900707, + "loss": 2.8078, + "theoretical_loss": 3.812374074658317, + "tokens_seen": 643104768 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004066198595787362, + "loss": 2.8899, + "theoretical_loss": 3.812334031001764, + "tokens_seen": 643170304 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040660982948846543, + "loss": 2.6655, + "theoretical_loss": 3.812293992567602, + "tokens_seen": 643235840 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040659979939819456, + "loss": 2.7146, + "theoretical_loss": 3.8122539593546163, + "tokens_seen": 643301376 + }, + { + "epoch": 0.19, + "learning_rate": 0.0004065897693079238, + "loss": 2.9341, + "theoretical_loss": 3.8122139313615953, + "tokens_seen": 643366912 + }, + { + "epoch": 0.19, + "learning_rate": 0.000406579739217653, + "loss": 2.856, + "theoretical_loss": 3.8121739085873267, + "tokens_seen": 643432448 + }, + { + "epoch": 0.19, + "learning_rate": 0.00040656970912738216, + "loss": 2.7503, + "theoretical_loss": 3.812133891030598, + "tokens_seen": 643497984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040655967903711134, + "loss": 2.8333, + "theoretical_loss": 3.812093878690199, + "tokens_seen": 643563520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065496489468405, + "loss": 2.6505, + "theoretical_loss": 3.8120538715649177, + "tokens_seen": 643629056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065396188565697, + "loss": 2.6991, + "theoretical_loss": 3.8120138696535437, + "tokens_seen": 643694592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040652958876629894, + "loss": 2.8435, + "theoretical_loss": 3.8119738729548667, + "tokens_seen": 643760128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040651955867602806, + "loss": 2.9121, + "theoretical_loss": 3.811933881467677, + "tokens_seen": 643825664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 364407, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.769031047821045, + "objective/train/theoretical_loss": 3.8118938951907646, + "objective/train/tokens_used": 664351200, + "theoretical_loss": 3.8118938951907646, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004065095285857573, + "loss": 2.9188, + "theoretical_loss": 3.8118938951907646, + "tokens_seen": 643891200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064994984954865, + "loss": 3.0245, + "theoretical_loss": 3.8118539141229215, + "tokens_seen": 643956736 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040648946840521566, + "loss": 2.8493, + "theoretical_loss": 3.811813938262939, + "tokens_seen": 644022272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040647943831494484, + "loss": 2.7306, + "theoretical_loss": 3.811773967609608, + "tokens_seen": 644087808 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406469408224674, + "loss": 2.5553, + "theoretical_loss": 3.811734002161721, + "tokens_seen": 644153344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064593781344032, + "loss": 2.7385, + "theoretical_loss": 3.8116940419180705, + "tokens_seen": 644218880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040644934804413244, + "loss": 2.7768, + "theoretical_loss": 3.81165408687745, + "tokens_seen": 644284416 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040643931795386157, + "loss": 2.9223, + "theoretical_loss": 3.8116141370386525, + "tokens_seen": 644349952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004064292878635908, + "loss": 2.5762, + "theoretical_loss": 3.811574192400472, + "tokens_seen": 644415488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040641925777331993, + "loss": 2.6792, + "theoretical_loss": 3.8115342529617022, + "tokens_seen": 644481024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040640922768304916, + "loss": 2.7865, + "theoretical_loss": 3.811494318721139, + "tokens_seen": 644546560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040639919759277834, + "loss": 2.9013, + "theoretical_loss": 3.811454389677576, + "tokens_seen": 644612096 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063891675025075, + "loss": 2.5841, + "theoretical_loss": 3.811414465829809, + "tokens_seen": 644677632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063791374122367, + "loss": 2.7939, + "theoretical_loss": 3.8113745471766336, + "tokens_seen": 644743168 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063691073219659, + "loss": 3.0081, + "theoretical_loss": 3.811334633716847, + "tokens_seen": 644808704 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040635907723169507, + "loss": 2.5528, + "theoretical_loss": 3.811294725449245, + "tokens_seen": 644874240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004063490471414243, + "loss": 2.7421, + "theoretical_loss": 3.811254822372624, + "tokens_seen": 644939776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040633901705115343, + "loss": 2.7564, + "theoretical_loss": 3.811214924485783, + "tokens_seen": 645005312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040632898696088267, + "loss": 2.8748, + "theoretical_loss": 3.811175031787519, + "tokens_seen": 645070848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040631895687061185, + "loss": 2.7806, + "theoretical_loss": 3.81113514427663, + "tokens_seen": 645136384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040630892678034103, + "loss": 2.9969, + "theoretical_loss": 3.8110952619519143, + "tokens_seen": 645201920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062988966900702, + "loss": 2.7653, + "theoretical_loss": 3.811055384812172, + "tokens_seen": 645267456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062888665997994, + "loss": 2.8918, + "theoretical_loss": 3.8110155128562018, + "tokens_seen": 645332992 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040627883650952857, + "loss": 2.7826, + "theoretical_loss": 3.8109756460828037, + "tokens_seen": 645398528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062688064192578, + "loss": 2.9599, + "theoretical_loss": 3.810935784490778, + "tokens_seen": 645464064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 365032, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9448306560516357, + "objective/train/theoretical_loss": 3.8108959280789243, + "objective/train/tokens_used": 665989600, + "theoretical_loss": 3.8108959280789243, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040625877632898693, + "loss": 2.8312, + "theoretical_loss": 3.8108959280789243, + "tokens_seen": 645529600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040624874623871617, + "loss": 2.8855, + "theoretical_loss": 3.8108560768460453, + "tokens_seen": 645595136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062387161484453, + "loss": 2.8467, + "theoretical_loss": 3.810816230790942, + "tokens_seen": 645660672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040622868605817453, + "loss": 2.6674, + "theoretical_loss": 3.8107763899124145, + "tokens_seen": 645726208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062186559679037, + "loss": 2.8258, + "theoretical_loss": 3.8107365542092673, + "tokens_seen": 645791744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004062086258776329, + "loss": 2.9253, + "theoretical_loss": 3.8106967236803015, + "tokens_seen": 645857280 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061985957873621, + "loss": 2.7386, + "theoretical_loss": 3.8106568983243205, + "tokens_seen": 645922816 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040618856569709126, + "loss": 2.7275, + "theoretical_loss": 3.8106170781401287, + "tokens_seen": 645988352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040617853560682044, + "loss": 2.8876, + "theoretical_loss": 3.8105772631265284, + "tokens_seen": 646053888 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040616850551654967, + "loss": 2.9051, + "theoretical_loss": 3.8105374532823246, + "tokens_seen": 646119424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061584754262788, + "loss": 2.9001, + "theoretical_loss": 3.8104976486063213, + "tokens_seen": 646184960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040614844533600804, + "loss": 2.9, + "theoretical_loss": 3.8104578490973244, + "tokens_seen": 646250496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040613841524573727, + "loss": 2.8382, + "theoretical_loss": 3.810418054754139, + "tokens_seen": 646316032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004061283851554664, + "loss": 2.8047, + "theoretical_loss": 3.81037826557557, + "tokens_seen": 646381568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040611835506519563, + "loss": 2.7919, + "theoretical_loss": 3.810338481560425, + "tokens_seen": 646447104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040610832497492476, + "loss": 2.9111, + "theoretical_loss": 3.8102987027075095, + "tokens_seen": 646512640 + }, + { + "epoch": 0.2, + "learning_rate": 0.000406098294884654, + "loss": 2.8586, + "theoretical_loss": 3.810258929015631, + "tokens_seen": 646578176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060882647943832, + "loss": 2.678, + "theoretical_loss": 3.8102191604835967, + "tokens_seen": 646643712 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040607823470411236, + "loss": 2.781, + "theoretical_loss": 3.8101793971102147, + "tokens_seen": 646709248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040606820461384154, + "loss": 2.914, + "theoretical_loss": 3.8101396388942925, + "tokens_seen": 646774784 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060581745235707, + "loss": 2.7807, + "theoretical_loss": 3.810099885834639, + "tokens_seen": 646840320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060481444332999, + "loss": 2.8564, + "theoretical_loss": 3.810060137930064, + "tokens_seen": 646905856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040603811434302914, + "loss": 2.7308, + "theoretical_loss": 3.810020395179375, + "tokens_seen": 646971392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040602808425275826, + "loss": 2.898, + "theoretical_loss": 3.8099806575813835, + "tokens_seen": 647036928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060180541624875, + "loss": 2.8808, + "theoretical_loss": 3.809940925134899, + "tokens_seen": 647102464 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 365530, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.693211317062378, + "objective/train/theoretical_loss": 3.809901197838732, + "objective/train/tokens_used": 667628000, + "theoretical_loss": 3.809901197838732, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004060080240722167, + "loss": 2.7837, + "theoretical_loss": 3.809901197838732, + "tokens_seen": 647168000 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040599799398194586, + "loss": 3.0024, + "theoretical_loss": 3.8098614756916933, + "tokens_seen": 647233536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040598796389167504, + "loss": 2.8533, + "theoretical_loss": 3.8098217586925944, + "tokens_seen": 647299072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059779338014042, + "loss": 2.7603, + "theoretical_loss": 3.8097820468402475, + "tokens_seen": 647364608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004059679037111334, + "loss": 2.6107, + "theoretical_loss": 3.8097423401334636, + "tokens_seen": 647430144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040595787362086264, + "loss": 2.7351, + "theoretical_loss": 3.809702638571056, + "tokens_seen": 647495680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040594784353059177, + "loss": 2.6653, + "theoretical_loss": 3.8096629421518378, + "tokens_seen": 647561216 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405937813440321, + "loss": 3.0185, + "theoretical_loss": 3.809623250874622, + "tokens_seen": 647626752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040592778335005013, + "loss": 2.7992, + "theoretical_loss": 3.8095835647382223, + "tokens_seen": 647692288 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040591775325977936, + "loss": 2.6663, + "theoretical_loss": 3.809543883741453, + "tokens_seen": 647757824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040590772316950854, + "loss": 2.6354, + "theoretical_loss": 3.809504207883128, + "tokens_seen": 647823360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058976930792377, + "loss": 2.7836, + "theoretical_loss": 3.8094645371620626, + "tokens_seen": 647888896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058876629889669, + "loss": 2.7201, + "theoretical_loss": 3.809424871577072, + "tokens_seen": 647954432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058776328986961, + "loss": 2.7792, + "theoretical_loss": 3.8093852111269726, + "tokens_seen": 648019968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040586760280842527, + "loss": 2.7333, + "theoretical_loss": 3.809345555810579, + "tokens_seen": 648085504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058575727181545, + "loss": 2.7829, + "theoretical_loss": 3.8093059056267085, + "tokens_seen": 648151040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040584754262788363, + "loss": 2.8198, + "theoretical_loss": 3.8092662605741783, + "tokens_seen": 648216576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040583751253761287, + "loss": 2.7442, + "theoretical_loss": 3.8092266206518044, + "tokens_seen": 648282112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040582748244734205, + "loss": 2.7689, + "theoretical_loss": 3.809186985858406, + "tokens_seen": 648347648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040581745235707123, + "loss": 2.9267, + "theoretical_loss": 3.8091473561928, + "tokens_seen": 648413184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004058074222668004, + "loss": 2.9537, + "theoretical_loss": 3.8091077316538056, + "tokens_seen": 648478720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057973921765296, + "loss": 2.9389, + "theoretical_loss": 3.809068112240241, + "tokens_seen": 648544256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040578736208625877, + "loss": 2.7254, + "theoretical_loss": 3.8090284979509255, + "tokens_seen": 648609792 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405777331995988, + "loss": 2.8984, + "theoretical_loss": 3.8089888887846786, + "tokens_seen": 648675328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040576730190571713, + "loss": 2.8705, + "theoretical_loss": 3.808949284740321, + "tokens_seen": 648740864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 366782, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.094092845916748, + "objective/train/theoretical_loss": 3.8089096858166718, + "objective/train/tokens_used": 669266400, + "theoretical_loss": 3.8089096858166718, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040575727181544637, + "loss": 2.7799, + "theoretical_loss": 3.8089096858166718, + "tokens_seen": 648806400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057472417251755, + "loss": 2.9153, + "theoretical_loss": 3.808870092012553, + "tokens_seen": 648871936 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040573721163490473, + "loss": 2.8236, + "theoretical_loss": 3.808830503326785, + "tokens_seen": 648937472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057271815446339, + "loss": 2.5253, + "theoretical_loss": 3.8087909197581897, + "tokens_seen": 649003008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057171514543631, + "loss": 3.0232, + "theoretical_loss": 3.8087513413055887, + "tokens_seen": 649068544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004057071213640923, + "loss": 2.9203, + "theoretical_loss": 3.8087117679678046, + "tokens_seen": 649134080 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040569709127382146, + "loss": 2.7979, + "theoretical_loss": 3.8086721997436603, + "tokens_seen": 649199616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040568706118355064, + "loss": 2.9185, + "theoretical_loss": 3.8086326366319785, + "tokens_seen": 649265152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056770310932799, + "loss": 2.5927, + "theoretical_loss": 3.808593078631583, + "tokens_seen": 649330688 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405667001003009, + "loss": 2.7122, + "theoretical_loss": 3.808553525741297, + "tokens_seen": 649396224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040565697091273824, + "loss": 2.9182, + "theoretical_loss": 3.808513977959946, + "tokens_seen": 649461760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056469408224674, + "loss": 2.899, + "theoretical_loss": 3.8084744352863535, + "tokens_seen": 649527296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056369107321966, + "loss": 2.7249, + "theoretical_loss": 3.808434897719345, + "tokens_seen": 649592832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004056268806419258, + "loss": 2.9955, + "theoretical_loss": 3.8083953652577462, + "tokens_seen": 649658368 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040561685055165496, + "loss": 2.591, + "theoretical_loss": 3.8083558379003826, + "tokens_seen": 649723904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040560682046138414, + "loss": 2.8997, + "theoretical_loss": 3.8083163156460804, + "tokens_seen": 649789440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055967903711134, + "loss": 2.8928, + "theoretical_loss": 3.808276798493666, + "tokens_seen": 649854976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055867602808425, + "loss": 2.8441, + "theoretical_loss": 3.808237286441967, + "tokens_seen": 649920512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040557673019057174, + "loss": 2.9768, + "theoretical_loss": 3.8081977794898103, + "tokens_seen": 649986048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040556670010030087, + "loss": 2.5429, + "theoretical_loss": 3.808158277636024, + "tokens_seen": 650051584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055566700100301, + "loss": 2.8073, + "theoretical_loss": 3.808118780879436, + "tokens_seen": 650117120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055466399197593, + "loss": 2.8952, + "theoretical_loss": 3.8080792892188753, + "tokens_seen": 650182656 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040553660982948846, + "loss": 2.7382, + "theoretical_loss": 3.8080398026531697, + "tokens_seen": 650248192 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040552657973921764, + "loss": 2.7826, + "theoretical_loss": 3.80800032118115, + "tokens_seen": 650313728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004055165496489469, + "loss": 2.7656, + "theoretical_loss": 3.807960844801644, + "tokens_seen": 650379264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 367488, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.001251459121704, + "objective/train/theoretical_loss": 3.8079213735134845, + "objective/train/tokens_used": 670904800, + "theoretical_loss": 3.8079213735134845, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405506519558676, + "loss": 2.835, + "theoretical_loss": 3.8079213735134845, + "tokens_seen": 650444800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040549648946840524, + "loss": 2.6539, + "theoretical_loss": 3.8078819073154997, + "tokens_seen": 650510336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040548645937813437, + "loss": 2.7046, + "theoretical_loss": 3.807842446206521, + "tokens_seen": 650575872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054764292878636, + "loss": 2.928, + "theoretical_loss": 3.8078029901853805, + "tokens_seen": 650641408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054663991975928, + "loss": 2.6514, + "theoretical_loss": 3.8077635392509084, + "tokens_seen": 650706944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040545636910732197, + "loss": 3.1162, + "theoretical_loss": 3.807724093401938, + "tokens_seen": 650772480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040544633901705115, + "loss": 2.582, + "theoretical_loss": 3.8076846526373007, + "tokens_seen": 650838016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040543630892678033, + "loss": 2.9161, + "theoretical_loss": 3.8076452169558306, + "tokens_seen": 650903552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004054262788365095, + "loss": 2.8318, + "theoretical_loss": 3.8076057863563593, + "tokens_seen": 650969088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040541624874623874, + "loss": 2.7556, + "theoretical_loss": 3.807566360837722, + "tokens_seen": 651034624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040540621865596787, + "loss": 2.7484, + "theoretical_loss": 3.807526940398751, + "tokens_seen": 651100160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053961885656971, + "loss": 2.7967, + "theoretical_loss": 3.807487525038282, + "tokens_seen": 651165696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053861584754263, + "loss": 2.9202, + "theoretical_loss": 3.8074481147551484, + "tokens_seen": 651231232 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040537612838515547, + "loss": 2.7285, + "theoretical_loss": 3.807408709548187, + "tokens_seen": 651296768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053660982948847, + "loss": 2.8267, + "theoretical_loss": 3.8073693094162318, + "tokens_seen": 651362304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040535606820461383, + "loss": 2.7598, + "theoretical_loss": 3.807329914358119, + "tokens_seen": 651427840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040534603811434307, + "loss": 2.7406, + "theoretical_loss": 3.807290524372686, + "tokens_seen": 651493376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040533600802407225, + "loss": 2.5667, + "theoretical_loss": 3.807251139458767, + "tokens_seen": 651558912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040532597793380143, + "loss": 2.6472, + "theoretical_loss": 3.8072117596152015, + "tokens_seen": 651624448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053159478435306, + "loss": 2.6189, + "theoretical_loss": 3.807172384840826, + "tokens_seen": 651689984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004053059177532598, + "loss": 2.5579, + "theoretical_loss": 3.807133015134478, + "tokens_seen": 651755520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040529588766298897, + "loss": 2.6739, + "theoretical_loss": 3.807093650494996, + "tokens_seen": 651821056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052858575727182, + "loss": 2.8714, + "theoretical_loss": 3.8070542909212186, + "tokens_seen": 651886592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040527582748244733, + "loss": 2.9979, + "theoretical_loss": 3.807014936411984, + "tokens_seen": 651952128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040526579739217657, + "loss": 2.6012, + "theoretical_loss": 3.806975586966133, + "tokens_seen": 652017664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 368763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9794094562530518, + "objective/train/theoretical_loss": 3.8069362425825037, + "objective/train/tokens_used": 672543200, + "theoretical_loss": 3.8069362425825037, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052557673019057, + "loss": 2.7193, + "theoretical_loss": 3.8069362425825037, + "tokens_seen": 652083200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040524573721163493, + "loss": 2.9792, + "theoretical_loss": 3.806896903259937, + "tokens_seen": 652148736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052357071213641, + "loss": 2.8212, + "theoretical_loss": 3.8068575689972737, + "tokens_seen": 652214272 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052256770310933, + "loss": 2.8342, + "theoretical_loss": 3.806818239793354, + "tokens_seen": 652279808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004052156469408225, + "loss": 2.4792, + "theoretical_loss": 3.8067789156470195, + "tokens_seen": 652345344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040520561685055166, + "loss": 3.0418, + "theoretical_loss": 3.8067395965571116, + "tokens_seen": 652410880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040519558676028084, + "loss": 2.7229, + "theoretical_loss": 3.8067002825224723, + "tokens_seen": 652476416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051855566700101, + "loss": 2.9192, + "theoretical_loss": 3.806660973541944, + "tokens_seen": 652541952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051755265797392, + "loss": 2.7193, + "theoretical_loss": 3.80662166961437, + "tokens_seen": 652607488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040516549648946844, + "loss": 2.719, + "theoretical_loss": 3.8065823707385924, + "tokens_seen": 652673024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051554663991976, + "loss": 2.6547, + "theoretical_loss": 3.8065430769134556, + "tokens_seen": 652738560 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051454363089268, + "loss": 2.7879, + "theoretical_loss": 3.8065037881378028, + "tokens_seen": 652804096 + }, + { + "epoch": 0.2, + "learning_rate": 0.000405135406218656, + "loss": 2.9661, + "theoretical_loss": 3.8064645044104792, + "tokens_seen": 652869632 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040512537612838516, + "loss": 2.6646, + "theoretical_loss": 3.8064252257303286, + "tokens_seen": 652935168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040511534603811434, + "loss": 2.6163, + "theoretical_loss": 3.8063859520961967, + "tokens_seen": 653000704 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004051053159478436, + "loss": 2.9179, + "theoretical_loss": 3.8063466835069284, + "tokens_seen": 653066240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050952858575727, + "loss": 2.8113, + "theoretical_loss": 3.8063074199613696, + "tokens_seen": 653131776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040508525576730194, + "loss": 2.7895, + "theoretical_loss": 3.806268161458367, + "tokens_seen": 653197312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040507522567703107, + "loss": 2.6098, + "theoretical_loss": 3.8062289079967666, + "tokens_seen": 653262848 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050651955867603, + "loss": 2.6069, + "theoretical_loss": 3.8061896595754154, + "tokens_seen": 653328384 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050551654964895, + "loss": 2.7038, + "theoretical_loss": 3.8061504161931605, + "tokens_seen": 653393920 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040504513540621866, + "loss": 2.5877, + "theoretical_loss": 3.8061111778488503, + "tokens_seen": 653459456 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040503510531594784, + "loss": 2.7855, + "theoretical_loss": 3.8060719445413325, + "tokens_seen": 653524992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050250752256771, + "loss": 2.6148, + "theoretical_loss": 3.8060327162694563, + "tokens_seen": 653590528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004050150451354062, + "loss": 2.8388, + "theoretical_loss": 3.8059934930320685, + "tokens_seen": 653656064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 369902, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6009209156036377, + "objective/train/theoretical_loss": 3.8059542748280206, + "objective/train/tokens_used": 674181600, + "theoretical_loss": 3.8059542748280206, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040500501504513544, + "loss": 2.5162, + "theoretical_loss": 3.8059542748280206, + "tokens_seen": 653721600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040499498495486457, + "loss": 2.681, + "theoretical_loss": 3.805915061656161, + "tokens_seen": 653787136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049849548645938, + "loss": 2.7516, + "theoretical_loss": 3.8058758535153396, + "tokens_seen": 653852672 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404974924774323, + "loss": 2.6446, + "theoretical_loss": 3.805836650404407, + "tokens_seen": 653918208 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040496489468405217, + "loss": 2.659, + "theoretical_loss": 3.8057974523222144, + "tokens_seen": 653983744 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040495486459378135, + "loss": 2.7514, + "theoretical_loss": 3.8057582592676127, + "tokens_seen": 654049280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040494483450351053, + "loss": 2.8106, + "theoretical_loss": 3.8057190712394524, + "tokens_seen": 654114816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049348044132397, + "loss": 2.5477, + "theoretical_loss": 3.805679888236587, + "tokens_seen": 654180352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040492477432296894, + "loss": 2.889, + "theoretical_loss": 3.805640710257867, + "tokens_seen": 654245888 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040491474423269807, + "loss": 2.886, + "theoretical_loss": 3.805601537302147, + "tokens_seen": 654311424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004049047141424273, + "loss": 2.6806, + "theoretical_loss": 3.8055623693682783, + "tokens_seen": 654376960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040489468405215643, + "loss": 2.8927, + "theoretical_loss": 3.8055232064551148, + "tokens_seen": 654442496 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040488465396188567, + "loss": 3.0574, + "theoretical_loss": 3.8054840485615102, + "tokens_seen": 654508032 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040487462387161485, + "loss": 2.7444, + "theoretical_loss": 3.8054448956863194, + "tokens_seen": 654573568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040486459378134403, + "loss": 2.9624, + "theoretical_loss": 3.8054057478283956, + "tokens_seen": 654639104 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048545636910732, + "loss": 2.9824, + "theoretical_loss": 3.805366604986595, + "tokens_seen": 654704640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040484453360080245, + "loss": 2.9506, + "theoretical_loss": 3.8053274671597714, + "tokens_seen": 654770176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048345035105316, + "loss": 2.8364, + "theoretical_loss": 3.8052883343467814, + "tokens_seen": 654835712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004048244734202608, + "loss": 2.8075, + "theoretical_loss": 3.805249206546481, + "tokens_seen": 654901248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040481444332998994, + "loss": 2.7262, + "theoretical_loss": 3.8052100837577263, + "tokens_seen": 654966784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040480441323971917, + "loss": 2.7897, + "theoretical_loss": 3.805170965979374, + "tokens_seen": 655032320 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040479438314944835, + "loss": 2.6729, + "theoretical_loss": 3.805131853210282, + "tokens_seen": 655097856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040478435305917753, + "loss": 2.7982, + "theoretical_loss": 3.8050927454493064, + "tokens_seen": 655163392 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047743229689067, + "loss": 2.6755, + "theoretical_loss": 3.8050536426953063, + "tokens_seen": 655228928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047642928786359, + "loss": 2.6891, + "theoretical_loss": 3.8050145449471398, + "tokens_seen": 655294464 + }, + { + "debugging/Self-BLEU-5": NaN, + "debugging/distinct-1-grams": 0.0, + "debugging/distinct-2-grams": 0.0, + "debugging/entropy-1-grams": 0.0, + "debugging/entropy-2-grams": 0.0, + "debugging/length": NaN, + "debugging/num_segments": 0, + "debugging/score": NaN, + "debugging/score_std": NaN, + "epoch": 0.2, + "objective/train/docs_used": 370281, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.031588315963745, + "objective/train/theoretical_loss": 3.8049754522036645, + "objective/train/tokens_used": 675820000, + "theoretical_loss": 3.8049754522036645, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047542627883651, + "loss": 2.844, + "theoretical_loss": 3.8049754522036645, + "tokens_seen": 655360000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047442326980943, + "loss": 2.7745, + "theoretical_loss": 3.8049363644637406, + "tokens_seen": 655425536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040473420260782344, + "loss": 2.6569, + "theoretical_loss": 3.8048972817262268, + "tokens_seen": 655491072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047241725175527, + "loss": 3.1308, + "theoretical_loss": 3.8048582039899834, + "tokens_seen": 655556608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004047141424272818, + "loss": 2.8553, + "theoretical_loss": 3.8048191312538697, + "tokens_seen": 655622144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040470411233701104, + "loss": 2.5904, + "theoretical_loss": 3.8047800635167466, + "tokens_seen": 655687680 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046940822467402, + "loss": 2.6663, + "theoretical_loss": 3.804741000777476, + "tokens_seen": 655753216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046840521564694, + "loss": 3.0068, + "theoretical_loss": 3.8047019430349174, + "tokens_seen": 655818752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046740220661986, + "loss": 2.9107, + "theoretical_loss": 3.8046628902879336, + "tokens_seen": 655884288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046639919759278, + "loss": 2.843, + "theoretical_loss": 3.8046238425353858, + "tokens_seen": 655949824 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404653961885657, + "loss": 2.7753, + "theoretical_loss": 3.804584799776137, + "tokens_seen": 656015360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046439317953862, + "loss": 2.8462, + "theoretical_loss": 3.80454576200905, + "tokens_seen": 656080896 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040463390170511536, + "loss": 2.6241, + "theoretical_loss": 3.8045067292329877, + "tokens_seen": 656146432 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040462387161484454, + "loss": 2.8776, + "theoretical_loss": 3.8044677014468133, + "tokens_seen": 656211968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046138415245738, + "loss": 2.8266, + "theoretical_loss": 3.804428678649391, + "tokens_seen": 656277504 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004046038114343029, + "loss": 2.913, + "theoretical_loss": 3.8043896608395844, + "tokens_seen": 656343040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040459378134403214, + "loss": 2.7933, + "theoretical_loss": 3.804350648016259, + "tokens_seen": 656408576 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040458375125376127, + "loss": 2.8499, + "theoretical_loss": 3.8043116401782795, + "tokens_seen": 656474112 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045737211634905, + "loss": 2.7959, + "theoretical_loss": 3.8042726373245115, + "tokens_seen": 656539648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045636910732197, + "loss": 2.9439, + "theoretical_loss": 3.8042336394538196, + "tokens_seen": 656605184 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040455366098294886, + "loss": 2.9103, + "theoretical_loss": 3.8041946465650707, + "tokens_seen": 656670720 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040454363089267804, + "loss": 2.5819, + "theoretical_loss": 3.8041556586571312, + "tokens_seen": 656736256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045336008024073, + "loss": 2.8263, + "theoretical_loss": 3.804116675728868, + "tokens_seen": 656801792 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004045235707121364, + "loss": 2.9633, + "theoretical_loss": 3.8040776977791486, + "tokens_seen": 656867328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040451354062186564, + "loss": 2.9724, + "theoretical_loss": 3.8040387248068397, + "tokens_seen": 656932864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 371094, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9736223220825195, + "objective/train/theoretical_loss": 3.8039997568108097, + "objective/train/tokens_used": 677458400, + "theoretical_loss": 3.8039997568108097, + "tokens_seen": 656998400 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040450351053159477, + "loss": 2.8304, + "theoretical_loss": 3.8039997568108097, + "tokens_seen": 656998400 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404493480441324, + "loss": 2.9945, + "theoretical_loss": 3.803960793789927, + "tokens_seen": 657063936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044834503510532, + "loss": 2.7311, + "theoretical_loss": 3.8039218357430604, + "tokens_seen": 657129472 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040447342026078237, + "loss": 2.7709, + "theoretical_loss": 3.8038828826690785, + "tokens_seen": 657195008 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040446339017051155, + "loss": 2.8639, + "theoretical_loss": 3.803843934566851, + "tokens_seen": 657260544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040445336008024073, + "loss": 2.6372, + "theoretical_loss": 3.8038049914352476, + "tokens_seen": 657326080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044433299899699, + "loss": 2.75, + "theoretical_loss": 3.803766053273139, + "tokens_seen": 657391616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040443329989969915, + "loss": 3.0652, + "theoretical_loss": 3.8037271200793947, + "tokens_seen": 657457152 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040442326980942827, + "loss": 2.8485, + "theoretical_loss": 3.803688191852886, + "tokens_seen": 657522688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004044132397191575, + "loss": 2.6473, + "theoretical_loss": 3.803649268592485, + "tokens_seen": 657588224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040440320962888663, + "loss": 2.7618, + "theoretical_loss": 3.8036103502970624, + "tokens_seen": 657653760 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040439317953861587, + "loss": 2.9614, + "theoretical_loss": 3.8035714369654903, + "tokens_seen": 657719296 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040438314944834505, + "loss": 2.8396, + "theoretical_loss": 3.8035325285966413, + "tokens_seen": 657784832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040437311935807423, + "loss": 2.9647, + "theoretical_loss": 3.8034936251893883, + "tokens_seen": 657850368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043630892678034, + "loss": 2.844, + "theoretical_loss": 3.8034547267426038, + "tokens_seen": 657915904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040435305917753265, + "loss": 2.7157, + "theoretical_loss": 3.803415833255162, + "tokens_seen": 657981440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004043430290872618, + "loss": 2.8558, + "theoretical_loss": 3.803376944725936, + "tokens_seen": 658046976 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404332998996991, + "loss": 2.6925, + "theoretical_loss": 3.803338061153801, + "tokens_seen": 658112512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040432296890672014, + "loss": 2.8953, + "theoretical_loss": 3.8032991825376303, + "tokens_seen": 658178048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040431293881644937, + "loss": 2.8478, + "theoretical_loss": 3.8032603088763, + "tokens_seen": 658243584 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040430290872617855, + "loss": 2.6926, + "theoretical_loss": 3.803221440168685, + "tokens_seen": 658309120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040429287863590773, + "loss": 2.7789, + "theoretical_loss": 3.8031825764136604, + "tokens_seen": 658374656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042828485456369, + "loss": 2.8289, + "theoretical_loss": 3.8031437176101033, + "tokens_seen": 658440192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042728184553661, + "loss": 2.8906, + "theoretical_loss": 3.80310486375689, + "tokens_seen": 658505728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042627883650953, + "loss": 2.972, + "theoretical_loss": 3.803066014852896, + "tokens_seen": 658571264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 371720, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1109814643859863, + "objective/train/theoretical_loss": 3.8030271708970003, + "objective/train/tokens_used": 679096800, + "theoretical_loss": 3.8030271708970003, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042527582748245, + "loss": 2.7763, + "theoretical_loss": 3.8030271708970003, + "tokens_seen": 658636800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040424272818455364, + "loss": 2.7485, + "theoretical_loss": 3.802988331888079, + "tokens_seen": 658702336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042326980942829, + "loss": 3.001, + "theoretical_loss": 3.8029494978250105, + "tokens_seen": 658767872 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404222668004012, + "loss": 2.9512, + "theoretical_loss": 3.802910668706673, + "tokens_seen": 658833408 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040421263791374124, + "loss": 2.7941, + "theoretical_loss": 3.802871844531945, + "tokens_seen": 658898944 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004042026078234704, + "loss": 2.8382, + "theoretical_loss": 3.802833025299706, + "tokens_seen": 658964480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041925777331996, + "loss": 3.1284, + "theoretical_loss": 3.802794211008835, + "tokens_seen": 659030016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041825476429288, + "loss": 2.8907, + "theoretical_loss": 3.802755401658211, + "tokens_seen": 659095552 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404172517552658, + "loss": 2.5927, + "theoretical_loss": 3.802716597246715, + "tokens_seen": 659161088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040416248746238714, + "loss": 2.8932, + "theoretical_loss": 3.802677797773227, + "tokens_seen": 659226624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041524573721164, + "loss": 2.766, + "theoretical_loss": 3.802639003236629, + "tokens_seen": 659292160 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041424272818455, + "loss": 2.8984, + "theoretical_loss": 3.8026002136358006, + "tokens_seen": 659357696 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040413239719157474, + "loss": 2.9041, + "theoretical_loss": 3.8025614289696232, + "tokens_seen": 659423232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041223671013039, + "loss": 2.748, + "theoretical_loss": 3.8025226492369804, + "tokens_seen": 659488768 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041123370110331, + "loss": 2.8719, + "theoretical_loss": 3.802483874436753, + "tokens_seen": 659554304 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004041023069207623, + "loss": 2.6527, + "theoretical_loss": 3.8024451045678243, + "tokens_seen": 659619840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040409227683049147, + "loss": 2.8325, + "theoretical_loss": 3.8024063396290773, + "tokens_seen": 659685376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040408224674022065, + "loss": 2.7487, + "theoretical_loss": 3.8023675796193945, + "tokens_seen": 659750912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040722166499499, + "loss": 2.8749, + "theoretical_loss": 3.8023288245376614, + "tokens_seen": 659816448 + }, + { + "epoch": 0.2, + "learning_rate": 0.000404062186559679, + "loss": 2.8572, + "theoretical_loss": 3.8022900743827606, + "tokens_seen": 659881984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040405215646940824, + "loss": 2.9801, + "theoretical_loss": 3.8022513291535764, + "tokens_seen": 659947520 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040404212637913737, + "loss": 2.92, + "theoretical_loss": 3.8022125888489944, + "tokens_seen": 660013056 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040320962888666, + "loss": 2.8742, + "theoretical_loss": 3.8021738534678997, + "tokens_seen": 660078592 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004040220661985958, + "loss": 2.8415, + "theoretical_loss": 3.8021351230091773, + "tokens_seen": 660144128 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040401203610832497, + "loss": 3.0197, + "theoretical_loss": 3.802096397471714, + "tokens_seen": 660209664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 373016, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.935034990310669, + "objective/train/theoretical_loss": 3.802057676854395, + "objective/train/tokens_used": 680735200, + "theoretical_loss": 3.802057676854395, + "tokens_seen": 660275200 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040400200601805415, + "loss": 2.6929, + "theoretical_loss": 3.802057676854395, + "tokens_seen": 660275200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039919759277834, + "loss": 2.8014, + "theoretical_loss": 3.802018961156108, + "tokens_seen": 660340736 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039819458375125, + "loss": 3.053, + "theoretical_loss": 3.801980250375739, + "tokens_seen": 660406272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040397191574724175, + "loss": 2.898, + "theoretical_loss": 3.801941544512176, + "tokens_seen": 660471808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039618856569709, + "loss": 2.8489, + "theoretical_loss": 3.8019028435643065, + "tokens_seen": 660537344 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039518555667001, + "loss": 2.8731, + "theoretical_loss": 3.8018641475310186, + "tokens_seen": 660602880 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004039418254764293, + "loss": 2.7511, + "theoretical_loss": 3.801825456411201, + "tokens_seen": 660668416 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040393179538615847, + "loss": 2.8653, + "theoretical_loss": 3.8017867702037424, + "tokens_seen": 660733952 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040392176529588765, + "loss": 2.764, + "theoretical_loss": 3.801748088907531, + "tokens_seen": 660799488 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040391173520561683, + "loss": 2.763, + "theoretical_loss": 3.8017094125214577, + "tokens_seen": 660865024 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040390170511534607, + "loss": 2.9756, + "theoretical_loss": 3.8016707410444117, + "tokens_seen": 660930560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040389167502507525, + "loss": 2.8566, + "theoretical_loss": 3.8016320744752834, + "tokens_seen": 660996096 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040388164493480443, + "loss": 2.8749, + "theoretical_loss": 3.8015934128129634, + "tokens_seen": 661061632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038716148445336, + "loss": 2.8313, + "theoretical_loss": 3.8015547560563423, + "tokens_seen": 661127168 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040386158475426285, + "loss": 2.8092, + "theoretical_loss": 3.801516104204312, + "tokens_seen": 661192704 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403851554663992, + "loss": 2.9406, + "theoretical_loss": 3.801477457255764, + "tokens_seen": 661258240 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004038415245737212, + "loss": 2.7659, + "theoretical_loss": 3.8014388152095906, + "tokens_seen": 661323776 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040383149448345034, + "loss": 2.9872, + "theoretical_loss": 3.8014001780646836, + "tokens_seen": 661389312 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040382146439317957, + "loss": 2.9488, + "theoretical_loss": 3.801361545819936, + "tokens_seen": 661454848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040381143430290875, + "loss": 2.9806, + "theoretical_loss": 3.8013229184742414, + "tokens_seen": 661520384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040380140421263794, + "loss": 2.9426, + "theoretical_loss": 3.801284296026492, + "tokens_seen": 661585920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037913741223671, + "loss": 2.6688, + "theoretical_loss": 3.8012456784755835, + "tokens_seen": 661651456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037813440320963, + "loss": 2.8687, + "theoretical_loss": 3.801207065820409, + "tokens_seen": 661716992 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037713139418255, + "loss": 2.9271, + "theoretical_loss": 3.8011684580598626, + "tokens_seen": 661782528 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037612838515547, + "loss": 2.7706, + "theoretical_loss": 3.8011298551928405, + "tokens_seen": 661848064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 374068, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0581512451171875, + "objective/train/theoretical_loss": 3.801091257218237, + "objective/train/tokens_used": 682373600, + "theoretical_loss": 3.801091257218237, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040375125376128384, + "loss": 2.7388, + "theoretical_loss": 3.801091257218237, + "tokens_seen": 661913600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037412236710131, + "loss": 2.5943, + "theoretical_loss": 3.801052664134948, + "tokens_seen": 661979136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037311935807422, + "loss": 2.9245, + "theoretical_loss": 3.80101407594187, + "tokens_seen": 662044672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040372116349047144, + "loss": 2.7558, + "theoretical_loss": 3.8009754926378982, + "tokens_seen": 662110208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037111334002006, + "loss": 2.9066, + "theoretical_loss": 3.8009369142219303, + "tokens_seen": 662175744 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004037011033099298, + "loss": 2.6965, + "theoretical_loss": 3.8008983406928634, + "tokens_seen": 662241280 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403691073219659, + "loss": 2.8863, + "theoretical_loss": 3.800859772049595, + "tokens_seen": 662306816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036810431293882, + "loss": 2.7285, + "theoretical_loss": 3.8008212082910218, + "tokens_seen": 662372352 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040367101303911734, + "loss": 2.8133, + "theoretical_loss": 3.8007826494160426, + "tokens_seen": 662437888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036609829488466, + "loss": 2.7862, + "theoretical_loss": 3.8007440954235565, + "tokens_seen": 662503424 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036509528585757, + "loss": 2.635, + "theoretical_loss": 3.8007055463124617, + "tokens_seen": 662568960 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040364092276830494, + "loss": 2.6339, + "theoretical_loss": 3.8006670020816578, + "tokens_seen": 662634496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036308926780341, + "loss": 2.8162, + "theoretical_loss": 3.800628462730044, + "tokens_seen": 662700032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036208625877633, + "loss": 2.8348, + "theoretical_loss": 3.8005899282565205, + "tokens_seen": 662765568 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004036108324974925, + "loss": 2.8832, + "theoretical_loss": 3.8005513986599873, + "tokens_seen": 662831104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040360080240722167, + "loss": 2.8673, + "theoretical_loss": 3.8005128739393452, + "tokens_seen": 662896640 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040359077231695085, + "loss": 3.0328, + "theoretical_loss": 3.8004743540934953, + "tokens_seen": 662962176 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035807422266801, + "loss": 2.6331, + "theoretical_loss": 3.8004358391213393, + "tokens_seen": 663027712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035707121364092, + "loss": 2.6823, + "theoretical_loss": 3.800397329021778, + "tokens_seen": 663093248 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040356068204613844, + "loss": 2.7886, + "theoretical_loss": 3.8003588237937143, + "tokens_seen": 663158784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040355065195586757, + "loss": 3.0434, + "theoretical_loss": 3.8003203234360505, + "tokens_seen": 663224320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035406218655968, + "loss": 2.8486, + "theoretical_loss": 3.800281827947689, + "tokens_seen": 663289856 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403530591775326, + "loss": 2.8891, + "theoretical_loss": 3.800243337327533, + "tokens_seen": 663355392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040352056168505517, + "loss": 2.98, + "theoretical_loss": 3.8002048515744864, + "tokens_seen": 663420928 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040351053159478435, + "loss": 3.0469, + "theoretical_loss": 3.800166370687453, + "tokens_seen": 663486464 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 374525, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.942661762237549, + "objective/train/theoretical_loss": 3.8001278946653363, + "objective/train/tokens_used": 684012000, + "theoretical_loss": 3.8001278946653363, + "tokens_seen": 663552000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004035005015045136, + "loss": 2.7616, + "theoretical_loss": 3.8001278946653363, + "tokens_seen": 663552000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034904714142427, + "loss": 2.6917, + "theoretical_loss": 3.8000894235070417, + "tokens_seen": 663617536 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040348044132397195, + "loss": 2.8898, + "theoretical_loss": 3.800050957211474, + "tokens_seen": 663683072 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034704112337011, + "loss": 2.6352, + "theoretical_loss": 3.800012495777538, + "tokens_seen": 663748608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034603811434303, + "loss": 2.9331, + "theoretical_loss": 3.7999740392041392, + "tokens_seen": 663814144 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034503510531595, + "loss": 2.8874, + "theoretical_loss": 3.799935587490185, + "tokens_seen": 663879680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040344032096288867, + "loss": 3.0548, + "theoretical_loss": 3.7998971406345796, + "tokens_seen": 663945216 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040343029087261785, + "loss": 2.5845, + "theoretical_loss": 3.7998586986362315, + "tokens_seen": 664010752 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040342026078234703, + "loss": 2.8137, + "theoretical_loss": 3.799820261494047, + "tokens_seen": 664076288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004034102306920762, + "loss": 2.8299, + "theoretical_loss": 3.7997818292069327, + "tokens_seen": 664141824 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040340020060180545, + "loss": 2.8644, + "theoretical_loss": 3.7997434017737985, + "tokens_seen": 664207360 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033901705115346, + "loss": 2.6154, + "theoretical_loss": 3.7997049791935504, + "tokens_seen": 664272896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033801404212638, + "loss": 2.739, + "theoretical_loss": 3.7996665614650977, + "tokens_seen": 664338432 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403370110330993, + "loss": 2.6332, + "theoretical_loss": 3.799628148587349, + "tokens_seen": 664403968 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033600802407222, + "loss": 2.6486, + "theoretical_loss": 3.799589740559214, + "tokens_seen": 664469504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040335005015045136, + "loss": 2.8741, + "theoretical_loss": 3.7995513373796017, + "tokens_seen": 664535040 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040334002006018054, + "loss": 3.0168, + "theoretical_loss": 3.799512939047422, + "tokens_seen": 664600576 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033299899699097, + "loss": 2.946, + "theoretical_loss": 3.7994745455615857, + "tokens_seen": 664666112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040331995987963895, + "loss": 2.8055, + "theoretical_loss": 3.7994361569210024, + "tokens_seen": 664731648 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004033099297893681, + "loss": 2.9599, + "theoretical_loss": 3.799397773124584, + "tokens_seen": 664797184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032998996990973, + "loss": 2.7457, + "theoretical_loss": 3.7993593941712405, + "tokens_seen": 664862720 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040328986960882644, + "loss": 2.9906, + "theoretical_loss": 3.7993210200598853, + "tokens_seen": 664928256 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032798395185557, + "loss": 2.7725, + "theoretical_loss": 3.7992826507894284, + "tokens_seen": 664993792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040326980942828486, + "loss": 2.564, + "theoretical_loss": 3.7992442863587836, + "tokens_seen": 665059328 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040325977933801404, + "loss": 2.7479, + "theoretical_loss": 3.7992059267668634, + "tokens_seen": 665124864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 375614, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8573155403137207, + "objective/train/theoretical_loss": 3.79916757201258, + "objective/train/tokens_used": 685650400, + "theoretical_loss": 3.79916757201258, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032497492477432, + "loss": 2.8858, + "theoretical_loss": 3.79916757201258, + "tokens_seen": 665190400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032397191574724, + "loss": 3.0415, + "theoretical_loss": 3.799129222094848, + "tokens_seen": 665255936 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032296890672016, + "loss": 3.0551, + "theoretical_loss": 3.7990908770125804, + "tokens_seen": 665321472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004032196589769308, + "loss": 2.8895, + "theoretical_loss": 3.799052536764691, + "tokens_seen": 665387008 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040320962888665995, + "loss": 2.7187, + "theoretical_loss": 3.7990142013500945, + "tokens_seen": 665452544 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031995987963892, + "loss": 2.8924, + "theoretical_loss": 3.7989758707677064, + "tokens_seen": 665518080 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040318956870611836, + "loss": 2.8798, + "theoretical_loss": 3.7989375450164413, + "tokens_seen": 665583616 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040317953861584754, + "loss": 2.915, + "theoretical_loss": 3.798899224095214, + "tokens_seen": 665649152 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031695085255767, + "loss": 2.8973, + "theoretical_loss": 3.798860908002941, + "tokens_seen": 665714688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031594784353059, + "loss": 2.725, + "theoretical_loss": 3.798822596738539, + "tokens_seen": 665780224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040314944834503514, + "loss": 2.9635, + "theoretical_loss": 3.7987842903009237, + "tokens_seen": 665845760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031394182547643, + "loss": 2.8181, + "theoretical_loss": 3.7987459886890127, + "tokens_seen": 665911296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031293881644935, + "loss": 2.9355, + "theoretical_loss": 3.798707691901722, + "tokens_seen": 665976832 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004031193580742227, + "loss": 2.6325, + "theoretical_loss": 3.798669399937971, + "tokens_seen": 666042368 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040310932798395187, + "loss": 2.7554, + "theoretical_loss": 3.7986311127966763, + "tokens_seen": 666107904 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040309929789368105, + "loss": 2.917, + "theoretical_loss": 3.798592830476757, + "tokens_seen": 666173440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030892678034103, + "loss": 2.6412, + "theoretical_loss": 3.79855455297713, + "tokens_seen": 666238976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030792377131394, + "loss": 2.8717, + "theoretical_loss": 3.7985162802967167, + "tokens_seen": 666304512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040306920762286864, + "loss": 2.7658, + "theoretical_loss": 3.798478012434435, + "tokens_seen": 666370048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040305917753259777, + "loss": 3.1829, + "theoretical_loss": 3.7984397493892055, + "tokens_seen": 666435584 + }, + { + "epoch": 0.2, + "learning_rate": 0.000403049147442327, + "loss": 2.9596, + "theoretical_loss": 3.798401491159947, + "tokens_seen": 666501120 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030391173520562, + "loss": 2.9012, + "theoretical_loss": 3.7983632377455807, + "tokens_seen": 666566656 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040302908726178537, + "loss": 2.7809, + "theoretical_loss": 3.7983249891450273, + "tokens_seen": 666632192 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040301905717151455, + "loss": 2.761, + "theoretical_loss": 3.7982867453572076, + "tokens_seen": 666697728 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004030090270812438, + "loss": 2.7542, + "theoretical_loss": 3.798248506381043, + "tokens_seen": 666763264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 376311, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.005213975906372, + "objective/train/theoretical_loss": 3.7982102722154556, + "objective/train/tokens_used": 687288800, + "theoretical_loss": 3.7982102722154556, + "tokens_seen": 666828800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029989969909729, + "loss": 2.8448, + "theoretical_loss": 3.7982102722154556, + "tokens_seen": 666828800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040298896690070215, + "loss": 2.6927, + "theoretical_loss": 3.7981720428593677, + "tokens_seen": 666894336 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029789368104313, + "loss": 2.7377, + "theoretical_loss": 3.798133818311701, + "tokens_seen": 666959872 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029689067201605, + "loss": 2.5689, + "theoretical_loss": 3.798095598571379, + "tokens_seen": 667025408 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029588766298897, + "loss": 2.8091, + "theoretical_loss": 3.798057383637324, + "tokens_seen": 667090944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040294884653961887, + "loss": 2.9873, + "theoretical_loss": 3.798019173508461, + "tokens_seen": 667156480 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040293881644934805, + "loss": 2.8216, + "theoretical_loss": 3.7979809681837127, + "tokens_seen": 667222016 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040292878635907723, + "loss": 2.8597, + "theoretical_loss": 3.7979427676620032, + "tokens_seen": 667287552 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004029187562688064, + "loss": 2.8129, + "theoretical_loss": 3.797904571942258, + "tokens_seen": 667353088 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040290872617853565, + "loss": 2.7172, + "theoretical_loss": 3.797866381023401, + "tokens_seen": 667418624 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028986960882648, + "loss": 2.6435, + "theoretical_loss": 3.7978281949043575, + "tokens_seen": 667484160 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402888665997994, + "loss": 2.6663, + "theoretical_loss": 3.797790013584054, + "tokens_seen": 667549696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028786359077232, + "loss": 2.9507, + "theoretical_loss": 3.7977518370614156, + "tokens_seen": 667615232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028686058174524, + "loss": 2.8432, + "theoretical_loss": 3.797713665335369, + "tokens_seen": 667680768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040285857572718156, + "loss": 2.7833, + "theoretical_loss": 3.797675498404841, + "tokens_seen": 667746304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040284854563691074, + "loss": 2.8119, + "theoretical_loss": 3.7976373362687577, + "tokens_seen": 667811840 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028385155466399, + "loss": 2.8527, + "theoretical_loss": 3.7975991789260473, + "tokens_seen": 667877376 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040282848545636915, + "loss": 2.8808, + "theoretical_loss": 3.7975610263756368, + "tokens_seen": 667942912 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028184553660983, + "loss": 2.6857, + "theoretical_loss": 3.797522878616455, + "tokens_seen": 668008448 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004028084252758275, + "loss": 2.9767, + "theoretical_loss": 3.7974847356474295, + "tokens_seen": 668073984 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040279839518555664, + "loss": 2.8602, + "theoretical_loss": 3.797446597467489, + "tokens_seen": 668139520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027883650952859, + "loss": 2.8874, + "theoretical_loss": 3.797408464075563, + "tokens_seen": 668205056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040277833500501506, + "loss": 2.6017, + "theoretical_loss": 3.797370335470581, + "tokens_seen": 668270592 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040276830491474424, + "loss": 2.5136, + "theoretical_loss": 3.7973322116514723, + "tokens_seen": 668336128 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027582748244734, + "loss": 2.704, + "theoretical_loss": 3.797294092617167, + "tokens_seen": 668401664 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 377619, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8243908882141113, + "objective/train/theoretical_loss": 3.7972559783665965, + "objective/train/tokens_used": 688927200, + "theoretical_loss": 3.7972559783665965, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027482447342026, + "loss": 2.5602, + "theoretical_loss": 3.7972559783665965, + "tokens_seen": 668467200 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027382146439318, + "loss": 2.8683, + "theoretical_loss": 3.7972178688986897, + "tokens_seen": 668532736 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402728184553661, + "loss": 2.5877, + "theoretical_loss": 3.7971797642123786, + "tokens_seen": 668598272 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040271815446339015, + "loss": 2.9583, + "theoretical_loss": 3.7971416643065954, + "tokens_seen": 668663808 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004027081243731194, + "loss": 2.7348, + "theoretical_loss": 3.797103569180271, + "tokens_seen": 668729344 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040269809428284856, + "loss": 2.7532, + "theoretical_loss": 3.7970654788323377, + "tokens_seen": 668794880 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040268806419257774, + "loss": 2.9621, + "theoretical_loss": 3.797027393261728, + "tokens_seen": 668860416 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026780341023069, + "loss": 2.8518, + "theoretical_loss": 3.7969893124673746, + "tokens_seen": 668925952 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026680040120361, + "loss": 2.7747, + "theoretical_loss": 3.7969512364482116, + "tokens_seen": 668991488 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026579739217653, + "loss": 2.7672, + "theoretical_loss": 3.796913165203171, + "tokens_seen": 669057024 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026479438314945, + "loss": 2.896, + "theoretical_loss": 3.7968750987311877, + "tokens_seen": 669122560 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040263791374122365, + "loss": 2.9705, + "theoretical_loss": 3.796837037031196, + "tokens_seen": 669188096 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040263791374122365, + "loss": 3.0305, + "theoretical_loss": 3.796798980102129, + "tokens_seen": 669253632 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004026278836509529, + "loss": 2.8599, + "theoretical_loss": 3.7967609279429233, + "tokens_seen": 669319168 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402617853560682, + "loss": 2.8396, + "theoretical_loss": 3.7967228805525135, + "tokens_seen": 669384704 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040260782347041125, + "loss": 2.7468, + "theoretical_loss": 3.796684837929835, + "tokens_seen": 669450240 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040259779338014043, + "loss": 2.9226, + "theoretical_loss": 3.796646800073824, + "tokens_seen": 669515776 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025877632898696, + "loss": 2.8581, + "theoretical_loss": 3.796608766983416, + "tokens_seen": 669581312 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025777331995988, + "loss": 2.9392, + "theoretical_loss": 3.7965707386575485, + "tokens_seen": 669646848 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040256770310932797, + "loss": 2.8257, + "theoretical_loss": 3.796532715095158, + "tokens_seen": 669712384 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040255767301905715, + "loss": 2.7459, + "theoretical_loss": 3.7964946962951815, + "tokens_seen": 669777920 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025476429287864, + "loss": 3.0115, + "theoretical_loss": 3.7964566822565575, + "tokens_seen": 669843456 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025376128385155, + "loss": 2.7659, + "theoretical_loss": 3.796418672978223, + "tokens_seen": 669908992 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040252758274824475, + "loss": 2.7765, + "theoretical_loss": 3.7963806684591166, + "tokens_seen": 669974528 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040251755265797393, + "loss": 2.9028, + "theoretical_loss": 3.7963426686981774, + "tokens_seen": 670040064 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 378282, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.932396650314331, + "objective/train/theoretical_loss": 3.796304673694344, + "objective/train/tokens_used": 690565600, + "theoretical_loss": 3.796304673694344, + "tokens_seen": 670105600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004025075225677031, + "loss": 2.9771, + "theoretical_loss": 3.796304673694344, + "tokens_seen": 670105600 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024974924774323, + "loss": 2.7668, + "theoretical_loss": 3.7962666834465555, + "tokens_seen": 670171136 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024874623871615, + "loss": 2.634, + "theoretical_loss": 3.796228697953752, + "tokens_seen": 670236672 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040247743229689066, + "loss": 2.9584, + "theoretical_loss": 3.7961907172148726, + "tokens_seen": 670302208 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024674022066199, + "loss": 2.8543, + "theoretical_loss": 3.7961527412288585, + "tokens_seen": 670367744 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402457372116349, + "loss": 3.0343, + "theoretical_loss": 3.7961147699946505, + "tokens_seen": 670433280 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040244734202607825, + "loss": 2.8191, + "theoretical_loss": 3.796076803511189, + "tokens_seen": 670498816 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024373119358074, + "loss": 2.7031, + "theoretical_loss": 3.7960388417774156, + "tokens_seen": 670564352 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024272818455366, + "loss": 3.0333, + "theoretical_loss": 3.796000884792272, + "tokens_seen": 670629888 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004024172517552658, + "loss": 2.9055, + "theoretical_loss": 3.7959629325547004, + "tokens_seen": 670695424 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402407221664995, + "loss": 3.193, + "theoretical_loss": 3.7959249850636425, + "tokens_seen": 670760960 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023971915747242, + "loss": 2.7088, + "theoretical_loss": 3.795887042318042, + "tokens_seen": 670826496 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023871614844534, + "loss": 2.6832, + "theoretical_loss": 3.7958491043168414, + "tokens_seen": 670892032 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023771313941826, + "loss": 2.6732, + "theoretical_loss": 3.7958111710589844, + "tokens_seen": 670957568 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040236710130391176, + "loss": 2.7021, + "theoretical_loss": 3.7957732425434143, + "tokens_seen": 671023104 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040235707121364094, + "loss": 2.4503, + "theoretical_loss": 3.7957353187690748, + "tokens_seen": 671088640 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023470411233701, + "loss": 2.8226, + "theoretical_loss": 3.7956973997349115, + "tokens_seen": 671154176 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040233701103309935, + "loss": 3.0277, + "theoretical_loss": 3.7956594854398684, + "tokens_seen": 671219712 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023269809428285, + "loss": 2.7509, + "theoretical_loss": 3.7956215758828904, + "tokens_seen": 671285248 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004023169508525577, + "loss": 2.7952, + "theoretical_loss": 3.795583671062923, + "tokens_seen": 671350784 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040230692076228684, + "loss": 2.8198, + "theoretical_loss": 3.7955457709789124, + "tokens_seen": 671416320 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022968906720161, + "loss": 2.8985, + "theoretical_loss": 3.795507875629805, + "tokens_seen": 671481856 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040228686058174526, + "loss": 2.5066, + "theoretical_loss": 3.7954699850145457, + "tokens_seen": 671547392 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040227683049147444, + "loss": 2.6576, + "theoretical_loss": 3.795432099132083, + "tokens_seen": 671612928 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022668004012036, + "loss": 2.6896, + "theoretical_loss": 3.7953942179813627, + "tokens_seen": 671678464 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 379317, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.70872163772583, + "objective/train/theoretical_loss": 3.7953563415613325, + "objective/train/tokens_used": 692204000, + "theoretical_loss": 3.7953563415613325, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022567703109328, + "loss": 2.8489, + "theoretical_loss": 3.7953563415613325, + "tokens_seen": 671744000 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402246740220662, + "loss": 2.6866, + "theoretical_loss": 3.795318469870941, + "tokens_seen": 671809536 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022367101303912, + "loss": 3.0096, + "theoretical_loss": 3.795280602909135, + "tokens_seen": 671875072 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040222668004012035, + "loss": 2.771, + "theoretical_loss": 3.7952427406748646, + "tokens_seen": 671940608 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004022166499498496, + "loss": 2.5575, + "theoretical_loss": 3.7952048831670773, + "tokens_seen": 672006144 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040220661985957876, + "loss": 2.7143, + "theoretical_loss": 3.795167030384722, + "tokens_seen": 672071680 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040219658976930794, + "loss": 2.7934, + "theoretical_loss": 3.7951291823267495, + "tokens_seen": 672137216 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021865596790371, + "loss": 2.986, + "theoretical_loss": 3.7950913389921084, + "tokens_seen": 672202752 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021765295887663, + "loss": 2.8392, + "theoretical_loss": 3.7950535003797494, + "tokens_seen": 672268288 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021664994984955, + "loss": 2.7923, + "theoretical_loss": 3.795015666488623, + "tokens_seen": 672333824 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021564694082247, + "loss": 2.7625, + "theoretical_loss": 3.7949778373176795, + "tokens_seen": 672399360 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040214643931795385, + "loss": 2.8883, + "theoretical_loss": 3.7949400128658706, + "tokens_seen": 672464896 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021364092276831, + "loss": 2.8509, + "theoretical_loss": 3.7949021931321476, + "tokens_seen": 672530432 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004021263791374122, + "loss": 2.662, + "theoretical_loss": 3.794864378115462, + "tokens_seen": 672595968 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040211634904714145, + "loss": 2.6493, + "theoretical_loss": 3.7948265678147663, + "tokens_seen": 672661504 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040210631895687063, + "loss": 2.8812, + "theoretical_loss": 3.7947887622290133, + "tokens_seen": 672727040 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020962888665998, + "loss": 2.9279, + "theoretical_loss": 3.7947509613571553, + "tokens_seen": 672792576 + }, + { + "epoch": 0.2, + "learning_rate": 0.000402086258776329, + "loss": 2.8662, + "theoretical_loss": 3.7947131651981447, + "tokens_seen": 672858112 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040207622868605817, + "loss": 2.9209, + "theoretical_loss": 3.7946753737509367, + "tokens_seen": 672923648 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040206619859578735, + "loss": 2.86, + "theoretical_loss": 3.794637587014484, + "tokens_seen": 672989184 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020561685055166, + "loss": 2.9311, + "theoretical_loss": 3.7945998049877403, + "tokens_seen": 673054720 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020461384152457, + "loss": 2.7814, + "theoretical_loss": 3.794562027669661, + "tokens_seen": 673120256 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040203610832497495, + "loss": 3.022, + "theoretical_loss": 3.7945242550592013, + "tokens_seen": 673185792 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040202607823470413, + "loss": 2.8168, + "theoretical_loss": 3.7944864871553152, + "tokens_seen": 673251328 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020160481444333, + "loss": 2.8123, + "theoretical_loss": 3.794448723956959, + "tokens_seen": 673316864 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 379824, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8459346294403076, + "objective/train/theoretical_loss": 3.794410965463088, + "objective/train/tokens_used": 693842400, + "theoretical_loss": 3.794410965463088, + "tokens_seen": 673382400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004020060180541625, + "loss": 2.9922, + "theoretical_loss": 3.794410965463088, + "tokens_seen": 673382400 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019959879638917, + "loss": 2.5647, + "theoretical_loss": 3.7943732116726583, + "tokens_seen": 673447936 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040198595787362086, + "loss": 2.6541, + "theoretical_loss": 3.794335462584627, + "tokens_seen": 673513472 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019759277833501, + "loss": 2.7915, + "theoretical_loss": 3.7942977181979503, + "tokens_seen": 673579008 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019658976930792, + "loss": 2.7506, + "theoretical_loss": 3.794259978511586, + "tokens_seen": 673644544 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040195586760280845, + "loss": 2.8794, + "theoretical_loss": 3.794222243524491, + "tokens_seen": 673710080 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019458375125376, + "loss": 2.9101, + "theoretical_loss": 3.7941845132356233, + "tokens_seen": 673775616 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019358074222668, + "loss": 3.0697, + "theoretical_loss": 3.7941467876439408, + "tokens_seen": 673841152 + }, + { + "epoch": 0.2, + "learning_rate": 0.000401925777331996, + "loss": 3.0465, + "theoretical_loss": 3.7941090667484025, + "tokens_seen": 673906688 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004019157472417252, + "loss": 2.7341, + "theoretical_loss": 3.794071350547967, + "tokens_seen": 673972224 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040190571715145436, + "loss": 2.8905, + "theoretical_loss": 3.794033639041593, + "tokens_seen": 674037760 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018956870611836, + "loss": 2.7838, + "theoretical_loss": 3.793995932228241, + "tokens_seen": 674103296 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018856569709127, + "loss": 2.8153, + "theoretical_loss": 3.7939582301068695, + "tokens_seen": 674168832 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040187562688064196, + "loss": 2.647, + "theoretical_loss": 3.79392053267644, + "tokens_seen": 674234368 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018655967903711, + "loss": 2.6909, + "theoretical_loss": 3.7938828399359115, + "tokens_seen": 674299904 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018555667001003, + "loss": 3.1, + "theoretical_loss": 3.7938451518842458, + "tokens_seen": 674365440 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018455366098295, + "loss": 2.9233, + "theoretical_loss": 3.793807468520404, + "tokens_seen": 674430976 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018355065195587, + "loss": 2.8234, + "theoretical_loss": 3.7937697898433473, + "tokens_seen": 674496512 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040182547642928786, + "loss": 2.998, + "theoretical_loss": 3.7937321158520376, + "tokens_seen": 674562048 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040181544633901704, + "loss": 2.9544, + "theoretical_loss": 3.7936944465454374, + "tokens_seen": 674627584 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004018054162487462, + "loss": 2.7967, + "theoretical_loss": 3.7936567819225084, + "tokens_seen": 674693120 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040179538615847546, + "loss": 2.8876, + "theoretical_loss": 3.7936191219822133, + "tokens_seen": 674758656 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017853560682046, + "loss": 2.907, + "theoretical_loss": 3.7935814667235164, + "tokens_seen": 674824192 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017753259779338, + "loss": 2.9187, + "theoretical_loss": 3.79354381614538, + "tokens_seen": 674889728 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040176529588766295, + "loss": 2.9948, + "theoretical_loss": 3.793506170246768, + "tokens_seen": 674955264 + }, + { + "epoch": 0.2, + "objective/train/docs_used": 381014, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8223540782928467, + "objective/train/theoretical_loss": 3.7934685290266454, + "objective/train/tokens_used": 695480800, + "theoretical_loss": 3.7934685290266454, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040176529588766295, + "loss": 2.8171, + "theoretical_loss": 3.7934685290266454, + "tokens_seen": 675020800 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017552657973922, + "loss": 2.9283, + "theoretical_loss": 3.7934308924839755, + "tokens_seen": 675086336 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040174523570712137, + "loss": 2.9153, + "theoretical_loss": 3.793393260617724, + "tokens_seen": 675151872 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040173520561685055, + "loss": 2.7576, + "theoretical_loss": 3.7933556334268554, + "tokens_seen": 675217408 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040172517552657973, + "loss": 2.7632, + "theoretical_loss": 3.793318010910335, + "tokens_seen": 675282944 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040171514543630896, + "loss": 2.6694, + "theoretical_loss": 3.7932803930671297, + "tokens_seen": 675348480 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004017051153460381, + "loss": 2.9775, + "theoretical_loss": 3.7932427798962043, + "tokens_seen": 675414016 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016950852557673, + "loss": 2.9839, + "theoretical_loss": 3.793205171396526, + "tokens_seen": 675479552 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040168505516549645, + "loss": 2.7652, + "theoretical_loss": 3.7931675675670604, + "tokens_seen": 675545088 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016750250752257, + "loss": 2.7479, + "theoretical_loss": 3.7931299684067756, + "tokens_seen": 675610624 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040166499498495487, + "loss": 2.564, + "theoretical_loss": 3.7930923739146394, + "tokens_seen": 675676160 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040165496489468405, + "loss": 2.5691, + "theoretical_loss": 3.7930547840896187, + "tokens_seen": 675741696 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016449348044133, + "loss": 2.9134, + "theoretical_loss": 3.7930171989306816, + "tokens_seen": 675807232 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004016349047141424, + "loss": 2.7036, + "theoretical_loss": 3.7929796184367968, + "tokens_seen": 675872768 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040162487462387165, + "loss": 2.6151, + "theoretical_loss": 3.7929420426069327, + "tokens_seen": 675938304 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040161484453360083, + "loss": 2.7893, + "theoretical_loss": 3.7929044714400586, + "tokens_seen": 676003840 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040160481444333, + "loss": 2.6284, + "theoretical_loss": 3.792866904935144, + "tokens_seen": 676069376 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015947843530592, + "loss": 2.7396, + "theoretical_loss": 3.792829343091158, + "tokens_seen": 676134912 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040158475426278837, + "loss": 2.8118, + "theoretical_loss": 3.7927917859070712, + "tokens_seen": 676200448 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040157472417251755, + "loss": 2.774, + "theoretical_loss": 3.792754233381854, + "tokens_seen": 676265984 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015646940822468, + "loss": 3.0124, + "theoretical_loss": 3.7927166855144763, + "tokens_seen": 676331520 + }, + { + "epoch": 0.2, + "learning_rate": 0.0004015546639919759, + "loss": 2.5057, + "theoretical_loss": 3.7926791423039097, + "tokens_seen": 676397056 + }, + { + "epoch": 0.2, + "learning_rate": 0.00040154463390170515, + "loss": 2.8492, + "theoretical_loss": 3.792641603749126, + "tokens_seen": 676462592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040153460381143433, + "loss": 2.8639, + "theoretical_loss": 3.7926040698490957, + "tokens_seen": 676528128 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015245737211635, + "loss": 3.042, + "theoretical_loss": 3.7925665406027917, + "tokens_seen": 676593664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 382043, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.4060447216033936, + "objective/train/theoretical_loss": 3.792529016009186, + "objective/train/tokens_used": 697119200, + "theoretical_loss": 3.792529016009186, + "tokens_seen": 676659200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015145436308927, + "loss": 2.8706, + "theoretical_loss": 3.792529016009186, + "tokens_seen": 676659200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004015045135406219, + "loss": 2.7768, + "theoretical_loss": 3.7924914960672513, + "tokens_seen": 676724736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040149448345035106, + "loss": 2.817, + "theoretical_loss": 3.792453980775961, + "tokens_seen": 676790272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014844533600803, + "loss": 2.7802, + "theoretical_loss": 3.7924164701342873, + "tokens_seen": 676855808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014744232698094, + "loss": 2.9134, + "theoretical_loss": 3.7923789641412045, + "tokens_seen": 676921344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040146439317953865, + "loss": 2.8138, + "theoretical_loss": 3.7923414627956866, + "tokens_seen": 676986880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014543630892678, + "loss": 2.6232, + "theoretical_loss": 3.792303966096708, + "tokens_seen": 677052416 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401444332998997, + "loss": 2.9834, + "theoretical_loss": 3.792266474043243, + "tokens_seen": 677117952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014343029087262, + "loss": 2.8613, + "theoretical_loss": 3.7922289866342664, + "tokens_seen": 677183488 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014242728184554, + "loss": 2.7659, + "theoretical_loss": 3.7921915038687537, + "tokens_seen": 677249024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040141424272818456, + "loss": 2.8298, + "theoretical_loss": 3.7921540257456803, + "tokens_seen": 677314560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004014042126379138, + "loss": 2.7942, + "theoretical_loss": 3.7921165522640226, + "tokens_seen": 677380096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013941825476429, + "loss": 2.4966, + "theoretical_loss": 3.792079083422756, + "tokens_seen": 677445632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040138415245737216, + "loss": 2.7792, + "theoretical_loss": 3.7920416192208575, + "tokens_seen": 677511168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013741223671013, + "loss": 2.6478, + "theoretical_loss": 3.7920041596573038, + "tokens_seen": 677576704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013640922768305, + "loss": 2.6984, + "theoretical_loss": 3.7919667047310726, + "tokens_seen": 677642240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013540621865597, + "loss": 2.6497, + "theoretical_loss": 3.7919292544411407, + "tokens_seen": 677707776 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013440320962889, + "loss": 2.7696, + "theoretical_loss": 3.791891808786487, + "tokens_seen": 677773312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040133400200601806, + "loss": 2.8313, + "theoretical_loss": 3.791854367766088, + "tokens_seen": 677838848 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040132397191574724, + "loss": 2.9008, + "theoretical_loss": 3.7918169313789236, + "tokens_seen": 677904384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004013139418254764, + "loss": 2.7586, + "theoretical_loss": 3.7917794996239724, + "tokens_seen": 677969920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040130391173520566, + "loss": 2.8737, + "theoretical_loss": 3.791742072500213, + "tokens_seen": 678035456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012938816449348, + "loss": 2.9405, + "theoretical_loss": 3.7917046500066256, + "tokens_seen": 678100992 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401283851554664, + "loss": 2.7992, + "theoretical_loss": 3.7916672321421894, + "tokens_seen": 678166528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040127382146439315, + "loss": 2.8993, + "theoretical_loss": 3.791629818905884, + "tokens_seen": 678232064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 382695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.529493808746338, + "objective/train/theoretical_loss": 3.7915924102966914, + "objective/train/tokens_used": 698757600, + "theoretical_loss": 3.7915924102966914, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012637913741224, + "loss": 2.7103, + "theoretical_loss": 3.7915924102966914, + "tokens_seen": 678297600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040125376128385157, + "loss": 2.852, + "theoretical_loss": 3.7915550063135908, + "tokens_seen": 678363136 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040124373119358075, + "loss": 3.0078, + "theoretical_loss": 3.7915176069555643, + "tokens_seen": 678428672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040123370110330993, + "loss": 2.9411, + "theoretical_loss": 3.791480212221593, + "tokens_seen": 678494208 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040122367101303916, + "loss": 2.9414, + "theoretical_loss": 3.7914428221106586, + "tokens_seen": 678559744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012136409227683, + "loss": 2.8493, + "theoretical_loss": 3.791405436621743, + "tokens_seen": 678625280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004012036108324975, + "loss": 2.5601, + "theoretical_loss": 3.7913680557538285, + "tokens_seen": 678690816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040119358074222665, + "loss": 2.6255, + "theoretical_loss": 3.791330679505898, + "tokens_seen": 678756352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011835506519559, + "loss": 2.9282, + "theoretical_loss": 3.791293307876935, + "tokens_seen": 678821888 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040117352056168507, + "loss": 2.784, + "theoretical_loss": 3.7912559408659217, + "tokens_seen": 678887424 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040116349047141425, + "loss": 2.7854, + "theoretical_loss": 3.7912185784718435, + "tokens_seen": 678952960 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040115346038114343, + "loss": 2.6421, + "theoretical_loss": 3.7911812206936824, + "tokens_seen": 679018496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011434302908726, + "loss": 2.9031, + "theoretical_loss": 3.7911438675304234, + "tokens_seen": 679084032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011334002006018, + "loss": 2.8137, + "theoretical_loss": 3.791106518981052, + "tokens_seen": 679149568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040112337011033103, + "loss": 3.0802, + "theoretical_loss": 3.7910691750445515, + "tokens_seen": 679215104 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040111334002006016, + "loss": 2.664, + "theoretical_loss": 3.7910318357199086, + "tokens_seen": 679280640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004011033099297894, + "loss": 2.9432, + "theoretical_loss": 3.790994501006109, + "tokens_seen": 679346176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010932798395185, + "loss": 2.9391, + "theoretical_loss": 3.7909571709021375, + "tokens_seen": 679411712 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040108324974924775, + "loss": 2.8087, + "theoretical_loss": 3.7909198454069806, + "tokens_seen": 679477248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040107321965897693, + "loss": 2.7766, + "theoretical_loss": 3.790882524519626, + "tokens_seen": 679542784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010631895687061, + "loss": 2.8428, + "theoretical_loss": 3.7908452082390585, + "tokens_seen": 679608320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010531594784353, + "loss": 2.7802, + "theoretical_loss": 3.7908078965642673, + "tokens_seen": 679673856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040104312938816453, + "loss": 2.659, + "theoretical_loss": 3.7907705894942385, + "tokens_seen": 679739392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040103309929789366, + "loss": 2.6857, + "theoretical_loss": 3.790733287027961, + "tokens_seen": 679804928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004010230692076229, + "loss": 2.7388, + "theoretical_loss": 3.790695989164422, + "tokens_seen": 679870464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 383786, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7080867290496826, + "objective/train/theoretical_loss": 3.7906586959026107, + "objective/train/tokens_used": 700396000, + "theoretical_loss": 3.7906586959026107, + "tokens_seen": 679936000 + }, + { + "epoch": 0.21, + "learning_rate": 0.000401013039117352, + "loss": 2.8253, + "theoretical_loss": 3.7906586959026107, + "tokens_seen": 679936000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040100300902708126, + "loss": 3.1717, + "theoretical_loss": 3.7906214072415154, + "tokens_seen": 680001536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040099297893681044, + "loss": 2.9097, + "theoretical_loss": 3.7905841231801256, + "tokens_seen": 680067072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009829488465396, + "loss": 2.8925, + "theoretical_loss": 3.790546843717431, + "tokens_seen": 680132608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009729187562688, + "loss": 2.6975, + "theoretical_loss": 3.79050956885242, + "tokens_seen": 680198144 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400962888665998, + "loss": 2.6732, + "theoretical_loss": 3.790472298584084, + "tokens_seen": 680263680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040095285857572716, + "loss": 2.906, + "theoretical_loss": 3.7904350329114127, + "tokens_seen": 680329216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009428284854564, + "loss": 2.7083, + "theoretical_loss": 3.7903977718333977, + "tokens_seen": 680394752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009327983951855, + "loss": 2.8424, + "theoretical_loss": 3.790360515349029, + "tokens_seen": 680460288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040092276830491476, + "loss": 2.9744, + "theoretical_loss": 3.7903232634572985, + "tokens_seen": 680525824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009127382146439, + "loss": 2.8977, + "theoretical_loss": 3.7902860161571974, + "tokens_seen": 680591360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004009027081243731, + "loss": 2.7069, + "theoretical_loss": 3.7902487734477184, + "tokens_seen": 680656896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040089267803410236, + "loss": 2.8793, + "theoretical_loss": 3.790211535327853, + "tokens_seen": 680722432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008826479438315, + "loss": 2.6679, + "theoretical_loss": 3.7901743017965943, + "tokens_seen": 680787968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008726178535607, + "loss": 2.8633, + "theoretical_loss": 3.7901370728529353, + "tokens_seen": 680853504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008625877632899, + "loss": 2.8139, + "theoretical_loss": 3.790099848495869, + "tokens_seen": 680919040 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008525576730191, + "loss": 2.8153, + "theoretical_loss": 3.790062628724389, + "tokens_seen": 680984576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040084252758274826, + "loss": 2.7675, + "theoretical_loss": 3.790025413537489, + "tokens_seen": 681050112 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040083249749247744, + "loss": 2.9715, + "theoretical_loss": 3.789988202934164, + "tokens_seen": 681115648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004008224674022066, + "loss": 2.936, + "theoretical_loss": 3.789950996913408, + "tokens_seen": 681181184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040081243731193586, + "loss": 2.655, + "theoretical_loss": 3.7899137954742153, + "tokens_seen": 681246720 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400802407221665, + "loss": 3.0181, + "theoretical_loss": 3.7898765986155816, + "tokens_seen": 681312256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007923771313942, + "loss": 2.9044, + "theoretical_loss": 3.789839406336503, + "tokens_seen": 681377792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040078234704112335, + "loss": 2.8241, + "theoretical_loss": 3.789802218635974, + "tokens_seen": 681443328 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007723169508526, + "loss": 2.7947, + "theoretical_loss": 3.7897650355129917, + "tokens_seen": 681508864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 384361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9259161949157715, + "objective/train/theoretical_loss": 3.789727856966552, + "objective/train/tokens_used": 702034400, + "theoretical_loss": 3.789727856966552, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040076228686058177, + "loss": 2.7609, + "theoretical_loss": 3.789727856966552, + "tokens_seen": 681574400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040075225677031095, + "loss": 2.7081, + "theoretical_loss": 3.7896906829956523, + "tokens_seen": 681639936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040074222668004013, + "loss": 2.9535, + "theoretical_loss": 3.789653513599289, + "tokens_seen": 681705472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040073219658976936, + "loss": 2.7223, + "theoretical_loss": 3.7896163487764594, + "tokens_seen": 681771008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007221664994985, + "loss": 2.8175, + "theoretical_loss": 3.7895791885261616, + "tokens_seen": 681836544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004007121364092277, + "loss": 2.9393, + "theoretical_loss": 3.789542032847393, + "tokens_seen": 681902080 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040070210631895685, + "loss": 2.6259, + "theoretical_loss": 3.7895048817391532, + "tokens_seen": 681967616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006920762286861, + "loss": 2.7234, + "theoretical_loss": 3.78946773520044, + "tokens_seen": 682033152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040068204613841527, + "loss": 2.882, + "theoretical_loss": 3.789430593230252, + "tokens_seen": 682098688 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040067201604814445, + "loss": 2.5952, + "theoretical_loss": 3.7893934558275886, + "tokens_seen": 682164224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040066198595787363, + "loss": 2.914, + "theoretical_loss": 3.78935632299145, + "tokens_seen": 682229760 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006519558676028, + "loss": 2.7516, + "theoretical_loss": 3.789319194720836, + "tokens_seen": 682295296 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400641925777332, + "loss": 2.6412, + "theoretical_loss": 3.789282071014746, + "tokens_seen": 682360832 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040063189568706123, + "loss": 2.8659, + "theoretical_loss": 3.789244951872181, + "tokens_seen": 682426368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040062186559679036, + "loss": 2.6498, + "theoretical_loss": 3.7892078372921425, + "tokens_seen": 682491904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006118355065196, + "loss": 2.8014, + "theoretical_loss": 3.789170727273631, + "tokens_seen": 682557440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004006018054162487, + "loss": 2.7229, + "theoretical_loss": 3.789133621815648, + "tokens_seen": 682622976 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040059177532597795, + "loss": 2.7762, + "theoretical_loss": 3.7890965209171954, + "tokens_seen": 682688512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040058174523570713, + "loss": 2.7451, + "theoretical_loss": 3.7890594245772746, + "tokens_seen": 682754048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005717151454363, + "loss": 2.9726, + "theoretical_loss": 3.7890223327948895, + "tokens_seen": 682819584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005616850551655, + "loss": 2.6521, + "theoretical_loss": 3.788985245569042, + "tokens_seen": 682885120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040055165496489473, + "loss": 2.8672, + "theoretical_loss": 3.788948162898735, + "tokens_seen": 682950656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040054162487462386, + "loss": 2.7924, + "theoretical_loss": 3.7889110847829715, + "tokens_seen": 683016192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005315947843531, + "loss": 2.8288, + "theoretical_loss": 3.788874011220756, + "tokens_seen": 683081728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004005215646940822, + "loss": 2.8745, + "theoretical_loss": 3.788836942211092, + "tokens_seen": 683147264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 385388, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.051872491836548, + "objective/train/theoretical_loss": 3.7887998777529845, + "objective/train/tokens_used": 703672800, + "theoretical_loss": 3.7887998777529845, + "tokens_seen": 683212800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040051153460381146, + "loss": 2.8854, + "theoretical_loss": 3.7887998777529845, + "tokens_seen": 683212800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040050150451354064, + "loss": 3.0064, + "theoretical_loss": 3.7887628178454373, + "tokens_seen": 683278336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004914744232698, + "loss": 2.973, + "theoretical_loss": 3.7887257624874557, + "tokens_seen": 683343872 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400481444332999, + "loss": 2.9739, + "theoretical_loss": 3.7886887116780446, + "tokens_seen": 683409408 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004714142427282, + "loss": 2.628, + "theoretical_loss": 3.78865166541621, + "tokens_seen": 683474944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040046138415245736, + "loss": 2.7112, + "theoretical_loss": 3.7886146237009575, + "tokens_seen": 683540480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004513540621866, + "loss": 2.8486, + "theoretical_loss": 3.788577586531294, + "tokens_seen": 683606016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004413239719157, + "loss": 2.8971, + "theoretical_loss": 3.7885405539062242, + "tokens_seen": 683671552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040043129388164496, + "loss": 2.8651, + "theoretical_loss": 3.7885035258247566, + "tokens_seen": 683737088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004212637913741, + "loss": 2.8794, + "theoretical_loss": 3.788466502285898, + "tokens_seen": 683802624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004112337011033, + "loss": 2.769, + "theoretical_loss": 3.7884294832886547, + "tokens_seen": 683868160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004004012036108325, + "loss": 2.9099, + "theoretical_loss": 3.788392468832036, + "tokens_seen": 683933696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003911735205617, + "loss": 2.7388, + "theoretical_loss": 3.788355458915049, + "tokens_seen": 683999232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040038114343029087, + "loss": 3.0831, + "theoretical_loss": 3.7883184535367027, + "tokens_seen": 684064768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003711133400201, + "loss": 2.6536, + "theoretical_loss": 3.7882814526960047, + "tokens_seen": 684130304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040036108324974923, + "loss": 2.7046, + "theoretical_loss": 3.7882444563919657, + "tokens_seen": 684195840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040035105315947846, + "loss": 2.6015, + "theoretical_loss": 3.7882074646235933, + "tokens_seen": 684261376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003410230692076, + "loss": 2.7814, + "theoretical_loss": 3.7881704773898974, + "tokens_seen": 684326912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003309929789368, + "loss": 2.8598, + "theoretical_loss": 3.7881334946898892, + "tokens_seen": 684392448 + }, + { + "epoch": 0.21, + "learning_rate": 0.000400320962888666, + "loss": 2.7818, + "theoretical_loss": 3.7880965165225775, + "tokens_seen": 684457984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004003109327983952, + "loss": 2.5295, + "theoretical_loss": 3.7880595428869737, + "tokens_seen": 684523520 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040030090270812437, + "loss": 2.7813, + "theoretical_loss": 3.7880225737820883, + "tokens_seen": 684589056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040029087261785355, + "loss": 2.7439, + "theoretical_loss": 3.787985609206932, + "tokens_seen": 684654592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040028084252758273, + "loss": 2.8324, + "theoretical_loss": 3.7879486491605174, + "tokens_seen": 684720128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040027081243731197, + "loss": 2.7086, + "theoretical_loss": 3.7879116936418553, + "tokens_seen": 684785664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 386107, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4937210083007812, + "objective/train/theoretical_loss": 3.787874742649958, + "objective/train/tokens_used": 705311200, + "theoretical_loss": 3.787874742649958, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002607823470411, + "loss": 2.6333, + "theoretical_loss": 3.787874742649958, + "tokens_seen": 684851200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040025075225677033, + "loss": 2.9428, + "theoretical_loss": 3.787837796183838, + "tokens_seen": 684916736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040024072216649946, + "loss": 2.7115, + "theoretical_loss": 3.7878008542425086, + "tokens_seen": 684982272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004002306920762287, + "loss": 2.7586, + "theoretical_loss": 3.7877639168249817, + "tokens_seen": 685047808 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040022066198595787, + "loss": 2.7163, + "theoretical_loss": 3.7877269839302716, + "tokens_seen": 685113344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040021063189568705, + "loss": 2.6753, + "theoretical_loss": 3.787690055557391, + "tokens_seen": 685178880 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040020060180541623, + "loss": 2.8251, + "theoretical_loss": 3.7876531317053552, + "tokens_seen": 685244416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040019057171514547, + "loss": 2.8668, + "theoretical_loss": 3.7876162123731776, + "tokens_seen": 685309952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001805416248746, + "loss": 2.9311, + "theoretical_loss": 3.787579297559872, + "tokens_seen": 685375488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040017051153460383, + "loss": 2.853, + "theoretical_loss": 3.7875423872644545, + "tokens_seen": 685441024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040016048144433296, + "loss": 2.8186, + "theoretical_loss": 3.7875054814859404, + "tokens_seen": 685506560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001504513540622, + "loss": 2.6879, + "theoretical_loss": 3.787468580223344, + "tokens_seen": 685572096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040014042126379143, + "loss": 2.7141, + "theoretical_loss": 3.787431683475682, + "tokens_seen": 685637632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040013039117352056, + "loss": 2.6609, + "theoretical_loss": 3.7873947912419705, + "tokens_seen": 685703168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001203610832498, + "loss": 2.8451, + "theoretical_loss": 3.7873579035212255, + "tokens_seen": 685768704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004001103309929789, + "loss": 2.7973, + "theoretical_loss": 3.787321020312464, + "tokens_seen": 685834240 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040010030090270815, + "loss": 3.0104, + "theoretical_loss": 3.7872841416147036, + "tokens_seen": 685899776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040009027081243733, + "loss": 2.9192, + "theoretical_loss": 3.7872472674269604, + "tokens_seen": 685965312 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000802407221665, + "loss": 2.7897, + "theoretical_loss": 3.787210397748253, + "tokens_seen": 686030848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000702106318957, + "loss": 2.7675, + "theoretical_loss": 3.7871735325775986, + "tokens_seen": 686096384 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040006018054162493, + "loss": 2.7428, + "theoretical_loss": 3.7871366719140163, + "tokens_seen": 686161920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040005015045135406, + "loss": 2.7018, + "theoretical_loss": 3.787099815756524, + "tokens_seen": 686227456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000401203610833, + "loss": 2.7264, + "theoretical_loss": 3.787062964104141, + "tokens_seen": 686292992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004000300902708124, + "loss": 2.7911, + "theoretical_loss": 3.7870261169558868, + "tokens_seen": 686358528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040002006018054166, + "loss": 2.7299, + "theoretical_loss": 3.78698927431078, + "tokens_seen": 686424064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 387264, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.96657395362854, + "objective/train/theoretical_loss": 3.786952436167841, + "objective/train/tokens_used": 706949600, + "theoretical_loss": 3.786952436167841, + "tokens_seen": 686489600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00040001003009027084, + "loss": 2.6869, + "theoretical_loss": 3.786952436167841, + "tokens_seen": 686489600 + }, + { + "epoch": 0.21, + "learning_rate": 0.0004, + "loss": 2.8198, + "theoretical_loss": 3.78691560252609, + "tokens_seen": 686555136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999899699097292, + "loss": 2.5833, + "theoretical_loss": 3.7868787733845473, + "tokens_seen": 686620672 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999799398194584, + "loss": 2.6918, + "theoretical_loss": 3.786841948742233, + "tokens_seen": 686686208 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039996990972918756, + "loss": 2.823, + "theoretical_loss": 3.7868051285981696, + "tokens_seen": 686751744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999598796389168, + "loss": 2.8408, + "theoretical_loss": 3.7867683129513767, + "tokens_seen": 686817280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999498495486459, + "loss": 2.7538, + "theoretical_loss": 3.786731501800877, + "tokens_seen": 686882816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039993981945837516, + "loss": 2.8252, + "theoretical_loss": 3.7866946951456923, + "tokens_seen": 686948352 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999297893681043, + "loss": 2.6463, + "theoretical_loss": 3.786657892984845, + "tokens_seen": 687013888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999197592778335, + "loss": 2.8026, + "theoretical_loss": 3.786621095317357, + "tokens_seen": 687079424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003999097291875627, + "loss": 2.8123, + "theoretical_loss": 3.7865843021422516, + "tokens_seen": 687144960 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998996990972919, + "loss": 2.5796, + "theoretical_loss": 3.7865475134585522, + "tokens_seen": 687210496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039988966900702107, + "loss": 3.0825, + "theoretical_loss": 3.7865107292652818, + "tokens_seen": 687276032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998796389167503, + "loss": 2.7263, + "theoretical_loss": 3.786473949561465, + "tokens_seen": 687341568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039986960882647943, + "loss": 2.6653, + "theoretical_loss": 3.7864371743461254, + "tokens_seen": 687407104 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039985957873620866, + "loss": 2.7328, + "theoretical_loss": 3.786400403618287, + "tokens_seen": 687472640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998495486459378, + "loss": 2.5849, + "theoretical_loss": 3.7863636373769745, + "tokens_seen": 687538176 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399839518555667, + "loss": 2.6915, + "theoretical_loss": 3.7863268756212136, + "tokens_seen": 687603712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998294884653962, + "loss": 2.8043, + "theoretical_loss": 3.7862901183500295, + "tokens_seen": 687669248 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003998194583751254, + "loss": 2.9892, + "theoretical_loss": 3.7862533655624473, + "tokens_seen": 687734784 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039980942828485457, + "loss": 2.7182, + "theoretical_loss": 3.786216617257493, + "tokens_seen": 687800320 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039979939819458375, + "loss": 2.7629, + "theoretical_loss": 3.786179873434193, + "tokens_seen": 687865856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039978936810431293, + "loss": 2.4144, + "theoretical_loss": 3.7861431340915743, + "tokens_seen": 687931392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039977933801404217, + "loss": 2.7272, + "theoretical_loss": 3.7861063992286637, + "tokens_seen": 687996928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997693079237713, + "loss": 2.8178, + "theoretical_loss": 3.786069668844487, + "tokens_seen": 688062464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 387908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.436974287033081, + "objective/train/theoretical_loss": 3.786032942938073, + "objective/train/tokens_used": 708588000, + "theoretical_loss": 3.786032942938073, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039975927783350053, + "loss": 2.6594, + "theoretical_loss": 3.786032942938073, + "tokens_seen": 688128000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039974924774322966, + "loss": 2.7045, + "theoretical_loss": 3.785996221508449, + "tokens_seen": 688193536 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003997392176529589, + "loss": 2.6022, + "theoretical_loss": 3.7859595045546426, + "tokens_seen": 688259072 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039972918756268807, + "loss": 2.8448, + "theoretical_loss": 3.7859227920756835, + "tokens_seen": 688324608 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039971915747241725, + "loss": 2.8839, + "theoretical_loss": 3.785886084070599, + "tokens_seen": 688390144 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039970912738214643, + "loss": 2.8146, + "theoretical_loss": 3.7858493805384184, + "tokens_seen": 688455680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039969909729187567, + "loss": 2.8004, + "theoretical_loss": 3.7858126814781707, + "tokens_seen": 688521216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996890672016048, + "loss": 2.7867, + "theoretical_loss": 3.7857759868888863, + "tokens_seen": 688586752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039967903711133403, + "loss": 3.0736, + "theoretical_loss": 3.785739296769595, + "tokens_seen": 688652288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039966900702106316, + "loss": 2.7904, + "theoretical_loss": 3.7857026111193264, + "tokens_seen": 688717824 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996589769307924, + "loss": 2.7473, + "theoretical_loss": 3.785665929937111, + "tokens_seen": 688783360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996489468405216, + "loss": 2.8171, + "theoretical_loss": 3.7856292532219795, + "tokens_seen": 688848896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039963891675025076, + "loss": 2.9703, + "theoretical_loss": 3.7855925809729634, + "tokens_seen": 688914432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039962888665997994, + "loss": 2.3776, + "theoretical_loss": 3.7855559131890946, + "tokens_seen": 688979968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996188565697091, + "loss": 2.8586, + "theoretical_loss": 3.7855192498694032, + "tokens_seen": 689045504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003996088264794383, + "loss": 2.7614, + "theoretical_loss": 3.785482591012922, + "tokens_seen": 689111040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039959879638916754, + "loss": 2.5907, + "theoretical_loss": 3.785445936618684, + "tokens_seen": 689176576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039958876629889666, + "loss": 2.6694, + "theoretical_loss": 3.785409286685721, + "tokens_seen": 689242112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995787362086259, + "loss": 2.7639, + "theoretical_loss": 3.785372641213066, + "tokens_seen": 689307648 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995687061183551, + "loss": 2.7779, + "theoretical_loss": 3.7853360001997522, + "tokens_seen": 689373184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039955867602808426, + "loss": 2.5631, + "theoretical_loss": 3.7852993636448127, + "tokens_seen": 689438720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039954864593781344, + "loss": 2.7811, + "theoretical_loss": 3.7852627315472818, + "tokens_seen": 689504256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995386158475426, + "loss": 2.7612, + "theoretical_loss": 3.785226103906194, + "tokens_seen": 689569792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003995285857572718, + "loss": 2.7939, + "theoretical_loss": 3.7851894807205833, + "tokens_seen": 689635328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039951855566700104, + "loss": 2.8221, + "theoretical_loss": 3.7851528619894843, + "tokens_seen": 689700864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 389134, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.952434778213501, + "objective/train/theoretical_loss": 3.7851162477119313, + "objective/train/tokens_used": 710226400, + "theoretical_loss": 3.7851162477119313, + "tokens_seen": 689766400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039950852557673017, + "loss": 2.9159, + "theoretical_loss": 3.7851162477119313, + "tokens_seen": 689766400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994984954864594, + "loss": 2.7995, + "theoretical_loss": 3.7850796378869607, + "tokens_seen": 689831936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039948846539618853, + "loss": 2.595, + "theoretical_loss": 3.7850430325136077, + "tokens_seen": 689897472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039947843530591776, + "loss": 2.771, + "theoretical_loss": 3.7850064315909084, + "tokens_seen": 689963008 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039946840521564694, + "loss": 2.78, + "theoretical_loss": 3.7849698351178986, + "tokens_seen": 690028544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994583751253761, + "loss": 2.7472, + "theoretical_loss": 3.784933243093615, + "tokens_seen": 690094080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994483450351053, + "loss": 2.8655, + "theoretical_loss": 3.7848966555170946, + "tokens_seen": 690159616 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994383149448345, + "loss": 2.6616, + "theoretical_loss": 3.7848600723873744, + "tokens_seen": 690225152 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039942828485456367, + "loss": 2.6761, + "theoretical_loss": 3.7848234937034917, + "tokens_seen": 690290688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003994182547642929, + "loss": 2.7662, + "theoretical_loss": 3.784786919464484, + "tokens_seen": 690356224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039940822467402203, + "loss": 2.9131, + "theoretical_loss": 3.7847503496693893, + "tokens_seen": 690421760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039939819458375127, + "loss": 2.8552, + "theoretical_loss": 3.7847137843172467, + "tokens_seen": 690487296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993881644934805, + "loss": 2.823, + "theoretical_loss": 3.784677223407094, + "tokens_seen": 690552832 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039937813440320963, + "loss": 2.7278, + "theoretical_loss": 3.78464066693797, + "tokens_seen": 690618368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039936810431293886, + "loss": 3.1157, + "theoretical_loss": 3.7846041149089142, + "tokens_seen": 690683904 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399358074222668, + "loss": 2.9657, + "theoretical_loss": 3.784567567318967, + "tokens_seen": 690749440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993480441323972, + "loss": 2.7648, + "theoretical_loss": 3.7845310241671664, + "tokens_seen": 690814976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993380140421264, + "loss": 2.6964, + "theoretical_loss": 3.7844944854525537, + "tokens_seen": 690880512 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003993279839518556, + "loss": 2.8395, + "theoretical_loss": 3.7844579511741694, + "tokens_seen": 690946048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039931795386158477, + "loss": 2.7431, + "theoretical_loss": 3.784421421331053, + "tokens_seen": 691011584 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039930792377131395, + "loss": 2.8044, + "theoretical_loss": 3.784384895922247, + "tokens_seen": 691077120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039929789368104313, + "loss": 2.6512, + "theoretical_loss": 3.7843483749467914, + "tokens_seen": 691142656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039928786359077237, + "loss": 2.7691, + "theoretical_loss": 3.7843118584037287, + "tokens_seen": 691208192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992778335005015, + "loss": 2.7728, + "theoretical_loss": 3.7842753462921004, + "tokens_seen": 691273728 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039926780341023073, + "loss": 2.6623, + "theoretical_loss": 3.784238838610949, + "tokens_seen": 691339264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 389824, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.667067050933838, + "objective/train/theoretical_loss": 3.784202335359316, + "objective/train/tokens_used": 711864800, + "theoretical_loss": 3.784202335359316, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039925777331995986, + "loss": 2.8217, + "theoretical_loss": 3.784202335359316, + "tokens_seen": 691404800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003992477432296891, + "loss": 2.6805, + "theoretical_loss": 3.7841658365362454, + "tokens_seen": 691470336 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039923771313941827, + "loss": 2.8809, + "theoretical_loss": 3.78412934214078, + "tokens_seen": 691535872 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039922768304914745, + "loss": 2.7089, + "theoretical_loss": 3.784092852171962, + "tokens_seen": 691601408 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039921765295887663, + "loss": 2.7421, + "theoretical_loss": 3.784056366628837, + "tokens_seen": 691666944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039920762286860587, + "loss": 2.7233, + "theoretical_loss": 3.7840198855104474, + "tokens_seen": 691732480 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399197592778335, + "loss": 2.5454, + "theoretical_loss": 3.783983408815838, + "tokens_seen": 691798016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039918756268806423, + "loss": 2.7803, + "theoretical_loss": 3.7839469365440537, + "tokens_seen": 691863552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039917753259779336, + "loss": 2.9658, + "theoretical_loss": 3.783910468694139, + "tokens_seen": 691929088 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991675025075226, + "loss": 2.8263, + "theoretical_loss": 3.7838740052651394, + "tokens_seen": 691994624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991574724172518, + "loss": 2.8775, + "theoretical_loss": 3.783837546256099, + "tokens_seen": 692060160 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039914744232698096, + "loss": 2.5907, + "theoretical_loss": 3.7838010916660654, + "tokens_seen": 692125696 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039913741223671014, + "loss": 2.9517, + "theoretical_loss": 3.783764641494084, + "tokens_seen": 692191232 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991273821464393, + "loss": 2.7124, + "theoretical_loss": 3.783728195739201, + "tokens_seen": 692256768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003991173520561685, + "loss": 2.6897, + "theoretical_loss": 3.7836917544004627, + "tokens_seen": 692322304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039910732196589774, + "loss": 2.8147, + "theoretical_loss": 3.7836553174769163, + "tokens_seen": 692387840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039909729187562686, + "loss": 2.8122, + "theoretical_loss": 3.783618884967609, + "tokens_seen": 692453376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990872617853561, + "loss": 2.7954, + "theoretical_loss": 3.783582456871589, + "tokens_seen": 692518912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990772316950853, + "loss": 2.767, + "theoretical_loss": 3.7835460331879034, + "tokens_seen": 692584448 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039906720160481446, + "loss": 2.5768, + "theoretical_loss": 3.7835096139155997, + "tokens_seen": 692649984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039905717151454364, + "loss": 3.0041, + "theoretical_loss": 3.783473199053728, + "tokens_seen": 692715520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990471414242728, + "loss": 2.7621, + "theoretical_loss": 3.7834367886013354, + "tokens_seen": 692781056 + }, + { + "epoch": 0.21, + "learning_rate": 0.000399037111334002, + "loss": 2.6121, + "theoretical_loss": 3.7834003825574722, + "tokens_seen": 692846592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039902708124373124, + "loss": 2.6698, + "theoretical_loss": 3.7833639809211865, + "tokens_seen": 692912128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039901705115346037, + "loss": 2.7172, + "theoretical_loss": 3.7833275836915288, + "tokens_seen": 692977664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 391100, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.815354585647583, + "objective/train/theoretical_loss": 3.7832911908675486, + "objective/train/tokens_used": 713503200, + "theoretical_loss": 3.7832911908675486, + "tokens_seen": 693043200 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003990070210631896, + "loss": 2.8474, + "theoretical_loss": 3.7832911908675486, + "tokens_seen": 693043200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039899699097291873, + "loss": 2.557, + "theoretical_loss": 3.7832548024482957, + "tokens_seen": 693108736 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039898696088264796, + "loss": 2.4426, + "theoretical_loss": 3.783218418432822, + "tokens_seen": 693174272 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039897693079237714, + "loss": 3.0426, + "theoretical_loss": 3.7831820388201765, + "tokens_seen": 693239808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989669007021063, + "loss": 2.754, + "theoretical_loss": 3.783145663609411, + "tokens_seen": 693305344 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989568706118355, + "loss": 2.7348, + "theoretical_loss": 3.783109292799577, + "tokens_seen": 693370880 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989468405215647, + "loss": 2.7635, + "theoretical_loss": 3.7830729263897265, + "tokens_seen": 693436416 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039893681043129387, + "loss": 2.5699, + "theoretical_loss": 3.783036564378911, + "tokens_seen": 693501952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003989267803410231, + "loss": 2.8455, + "theoretical_loss": 3.7830002067661814, + "tokens_seen": 693567488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039891675025075223, + "loss": 2.8716, + "theoretical_loss": 3.782963853550593, + "tokens_seen": 693633024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039890672016048147, + "loss": 2.7498, + "theoretical_loss": 3.7829275047311968, + "tokens_seen": 693698560 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039889669007021065, + "loss": 2.733, + "theoretical_loss": 3.782891160307046, + "tokens_seen": 693764096 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039888665997993983, + "loss": 2.7969, + "theoretical_loss": 3.782854820277195, + "tokens_seen": 693829632 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398876629889669, + "loss": 2.7611, + "theoretical_loss": 3.7828184846406963, + "tokens_seen": 693895168 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988665997993982, + "loss": 2.7715, + "theoretical_loss": 3.7827821533966044, + "tokens_seen": 693960704 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039885656970912737, + "loss": 2.778, + "theoretical_loss": 3.782745826543974, + "tokens_seen": 694026240 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988465396188566, + "loss": 2.9197, + "theoretical_loss": 3.7827095040818586, + "tokens_seen": 694091776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039883650952858573, + "loss": 2.6453, + "theoretical_loss": 3.7826731860093146, + "tokens_seen": 694157312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039882647943831497, + "loss": 2.7944, + "theoretical_loss": 3.782636872325396, + "tokens_seen": 694222848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003988164493480441, + "loss": 2.958, + "theoretical_loss": 3.782600563029159, + "tokens_seen": 694288384 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039880641925777333, + "loss": 2.7868, + "theoretical_loss": 3.782564258119659, + "tokens_seen": 694353920 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987963891675025, + "loss": 2.7221, + "theoretical_loss": 3.7825279575959514, + "tokens_seen": 694419456 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987863590772317, + "loss": 2.7764, + "theoretical_loss": 3.782491661457094, + "tokens_seen": 694484992 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987763289869609, + "loss": 2.6213, + "theoretical_loss": 3.7824553697021415, + "tokens_seen": 694550528 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039876629889669006, + "loss": 2.5601, + "theoretical_loss": 3.782419082330153, + "tokens_seen": 694616064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 392291, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8680379390716553, + "objective/train/theoretical_loss": 3.7823827993401844, + "objective/train/tokens_used": 715141600, + "theoretical_loss": 3.7823827993401844, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039875626880641924, + "loss": 2.7737, + "theoretical_loss": 3.7823827993401844, + "tokens_seen": 694681600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039874623871614847, + "loss": 2.7944, + "theoretical_loss": 3.782346520731293, + "tokens_seen": 694747136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987362086258776, + "loss": 2.8212, + "theoretical_loss": 3.7823102465025373, + "tokens_seen": 694812672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039872617853560683, + "loss": 2.728, + "theoretical_loss": 3.7822739766529754, + "tokens_seen": 694878208 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398716148445336, + "loss": 2.824, + "theoretical_loss": 3.7822377111816654, + "tokens_seen": 694943744 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003987061183550652, + "loss": 2.633, + "theoretical_loss": 3.7822014500876664, + "tokens_seen": 695009280 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986960882647944, + "loss": 2.7526, + "theoretical_loss": 3.7821651933700364, + "tokens_seen": 695074816 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039868605817452356, + "loss": 2.6716, + "theoretical_loss": 3.7821289410278354, + "tokens_seen": 695140352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039867602808425274, + "loss": 2.8782, + "theoretical_loss": 3.7820926930601226, + "tokens_seen": 695205888 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398665997993982, + "loss": 2.7649, + "theoretical_loss": 3.7820564494659585, + "tokens_seen": 695271424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986559679037111, + "loss": 2.6169, + "theoretical_loss": 3.7820202102444025, + "tokens_seen": 695336960 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039864593781344034, + "loss": 2.7008, + "theoretical_loss": 3.7819839753945153, + "tokens_seen": 695402496 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986359077231695, + "loss": 2.9026, + "theoretical_loss": 3.7819477449153576, + "tokens_seen": 695468032 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003986258776328987, + "loss": 2.7479, + "theoretical_loss": 3.78191151880599, + "tokens_seen": 695533568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039861584754262794, + "loss": 2.8503, + "theoretical_loss": 3.7818752970654748, + "tokens_seen": 695599104 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039860581745235706, + "loss": 2.7765, + "theoretical_loss": 3.7818390796928725, + "tokens_seen": 695664640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985957873620863, + "loss": 2.7862, + "theoretical_loss": 3.781802866687246, + "tokens_seen": 695730176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985857572718155, + "loss": 2.9017, + "theoretical_loss": 3.7817666580476565, + "tokens_seen": 695795712 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039857572718154466, + "loss": 2.6179, + "theoretical_loss": 3.781730453773167, + "tokens_seen": 695861248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039856569709127384, + "loss": 2.6699, + "theoretical_loss": 3.78169425386284, + "tokens_seen": 695926784 + }, + { + "epoch": 0.21, + "learning_rate": 0.000398555667001003, + "loss": 2.706, + "theoretical_loss": 3.781658058315738, + "tokens_seen": 695992320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985456369107322, + "loss": 2.9331, + "theoretical_loss": 3.7816218671309256, + "tokens_seen": 696057856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039853560682046144, + "loss": 2.619, + "theoretical_loss": 3.781585680307466, + "tokens_seen": 696123392 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039852557673019057, + "loss": 2.5891, + "theoretical_loss": 3.7815494978444226, + "tokens_seen": 696188928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003985155466399198, + "loss": 2.785, + "theoretical_loss": 3.78151331974086, + "tokens_seen": 696254464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 392983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6465673446655273, + "objective/train/theoretical_loss": 3.781477145995842, + "objective/train/tokens_used": 716780000, + "theoretical_loss": 3.781477145995842, + "tokens_seen": 696320000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039850551654964893, + "loss": 2.5574, + "theoretical_loss": 3.781477145995842, + "tokens_seen": 696320000 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039849548645937816, + "loss": 2.7248, + "theoretical_loss": 3.781440976608434, + "tokens_seen": 696385536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039848545636910734, + "loss": 2.9657, + "theoretical_loss": 3.7814048115777013, + "tokens_seen": 696451072 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984754262788365, + "loss": 2.9466, + "theoretical_loss": 3.7813686509027082, + "tokens_seen": 696516608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984653961885657, + "loss": 2.8556, + "theoretical_loss": 3.7813324945825215, + "tokens_seen": 696582144 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984553660982949, + "loss": 2.7166, + "theoretical_loss": 3.7812963426162067, + "tokens_seen": 696647680 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039844533600802407, + "loss": 2.8797, + "theoretical_loss": 3.7812601950028295, + "tokens_seen": 696713216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003984353059177533, + "loss": 2.6726, + "theoretical_loss": 3.7812240517414573, + "tokens_seen": 696778752 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039842527582748243, + "loss": 2.865, + "theoretical_loss": 3.7811879128311556, + "tokens_seen": 696844288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039841524573721167, + "loss": 2.7871, + "theoretical_loss": 3.781151778270993, + "tokens_seen": 696909824 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039840521564694085, + "loss": 2.7724, + "theoretical_loss": 3.7811156480600356, + "tokens_seen": 696975360 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039839518555667003, + "loss": 2.6619, + "theoretical_loss": 3.781079522197352, + "tokens_seen": 697040896 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983851554663992, + "loss": 2.6867, + "theoretical_loss": 3.7810434006820097, + "tokens_seen": 697106432 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983751253761284, + "loss": 2.7643, + "theoretical_loss": 3.7810072835130764, + "tokens_seen": 697171968 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039836509528585757, + "loss": 2.6857, + "theoretical_loss": 3.7809711706896216, + "tokens_seen": 697237504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983550651955868, + "loss": 2.7306, + "theoretical_loss": 3.7809350622107134, + "tokens_seen": 697303040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039834503510531593, + "loss": 2.6711, + "theoretical_loss": 3.7808989580754213, + "tokens_seen": 697368576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039833500501504517, + "loss": 2.8561, + "theoretical_loss": 3.780862858282814, + "tokens_seen": 697434112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983249749247743, + "loss": 2.7013, + "theoretical_loss": 3.780826762831962, + "tokens_seen": 697499648 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039831494483450353, + "loss": 2.7573, + "theoretical_loss": 3.7807906717219346, + "tokens_seen": 697565184 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003983049147442327, + "loss": 2.8347, + "theoretical_loss": 3.7807545849518025, + "tokens_seen": 697630720 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982948846539619, + "loss": 2.6078, + "theoretical_loss": 3.780718502520636, + "tokens_seen": 697696256 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982848545636911, + "loss": 2.6766, + "theoretical_loss": 3.780682424427506, + "tokens_seen": 697761792 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039827482447342026, + "loss": 2.6125, + "theoretical_loss": 3.780646350671484, + "tokens_seen": 697827328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039826479438314944, + "loss": 2.8044, + "theoretical_loss": 3.78061028125164, + "tokens_seen": 697892864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 394024, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.852461338043213, + "objective/train/theoretical_loss": 3.7805742161670466, + "objective/train/tokens_used": 718418400, + "theoretical_loss": 3.7805742161670466, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039825476429287867, + "loss": 2.8367, + "theoretical_loss": 3.7805742161670466, + "tokens_seen": 697958400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982447342026078, + "loss": 2.7577, + "theoretical_loss": 3.7805381554167763, + "tokens_seen": 698023936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039823470411233703, + "loss": 2.5646, + "theoretical_loss": 3.7805020989999, + "tokens_seen": 698089472 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982246740220662, + "loss": 2.7665, + "theoretical_loss": 3.7804660469154907, + "tokens_seen": 698155008 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982146439317954, + "loss": 2.8406, + "theoretical_loss": 3.7804299991626222, + "tokens_seen": 698220544 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003982046138415246, + "loss": 2.6331, + "theoretical_loss": 3.780393955740367, + "tokens_seen": 698286080 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039819458375125376, + "loss": 2.8337, + "theoretical_loss": 3.7803579166477976, + "tokens_seen": 698351616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039818455366098294, + "loss": 2.6492, + "theoretical_loss": 3.7803218818839883, + "tokens_seen": 698417152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981745235707122, + "loss": 2.7657, + "theoretical_loss": 3.7802858514480135, + "tokens_seen": 698482688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981644934804413, + "loss": 2.8308, + "theoretical_loss": 3.780249825338947, + "tokens_seen": 698548224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039815446339017054, + "loss": 2.6012, + "theoretical_loss": 3.7802138035558634, + "tokens_seen": 698613760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039814443329989966, + "loss": 2.8382, + "theoretical_loss": 3.7801777860978367, + "tokens_seen": 698679296 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981344032096289, + "loss": 2.8067, + "theoretical_loss": 3.780141772963944, + "tokens_seen": 698744832 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003981243731193581, + "loss": 2.8721, + "theoretical_loss": 3.7801057641532587, + "tokens_seen": 698810368 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039811434302908726, + "loss": 2.538, + "theoretical_loss": 3.7800697596648565, + "tokens_seen": 698875904 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039810431293881644, + "loss": 2.9022, + "theoretical_loss": 3.7800337594978153, + "tokens_seen": 698941440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980942828485457, + "loss": 2.7558, + "theoretical_loss": 3.7799977636512088, + "tokens_seen": 699006976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980842527582748, + "loss": 2.975, + "theoretical_loss": 3.779961772124115, + "tokens_seen": 699072512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039807422266800404, + "loss": 2.7963, + "theoretical_loss": 3.7799257849156107, + "tokens_seen": 699138048 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039806419257773317, + "loss": 2.838, + "theoretical_loss": 3.7798898020247726, + "tokens_seen": 699203584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980541624874624, + "loss": 2.5871, + "theoretical_loss": 3.779853823450678, + "tokens_seen": 699269120 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980441323971916, + "loss": 2.8418, + "theoretical_loss": 3.779817849192405, + "tokens_seen": 699334656 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039803410230692077, + "loss": 2.5972, + "theoretical_loss": 3.7797818792490303, + "tokens_seen": 699400192 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039802407221664995, + "loss": 2.8504, + "theoretical_loss": 3.7797459136196334, + "tokens_seen": 699465728 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039801404212637913, + "loss": 2.5989, + "theoretical_loss": 3.7797099523032927, + "tokens_seen": 699531264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 394602, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6187872886657715, + "objective/train/theoretical_loss": 3.7796739952990865, + "objective/train/tokens_used": 720056800, + "theoretical_loss": 3.7796739952990865, + "tokens_seen": 699596800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003980040120361083, + "loss": 2.5868, + "theoretical_loss": 3.7796739952990865, + "tokens_seen": 699596800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039799398194583754, + "loss": 2.6251, + "theoretical_loss": 3.779638042606094, + "tokens_seen": 699662336 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039798395185556667, + "loss": 2.7199, + "theoretical_loss": 3.779602094223394, + "tokens_seen": 699727872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979739217652959, + "loss": 2.6618, + "theoretical_loss": 3.7795661501500675, + "tokens_seen": 699793408 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039796389167502503, + "loss": 2.7668, + "theoretical_loss": 3.779530210385193, + "tokens_seen": 699858944 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039795386158475427, + "loss": 2.4999, + "theoretical_loss": 3.7794942749278517, + "tokens_seen": 699924480 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039794383149448345, + "loss": 2.9616, + "theoretical_loss": 3.7794583437771236, + "tokens_seen": 699990016 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039793380140421263, + "loss": 2.9088, + "theoretical_loss": 3.779422416932089, + "tokens_seen": 700055552 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979237713139418, + "loss": 2.8126, + "theoretical_loss": 3.77938649439183, + "tokens_seen": 700121088 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039791374122367105, + "loss": 2.6539, + "theoretical_loss": 3.779350576155427, + "tokens_seen": 700186624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003979037111334002, + "loss": 2.8109, + "theoretical_loss": 3.779314662221962, + "tokens_seen": 700252160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978936810431294, + "loss": 2.6834, + "theoretical_loss": 3.7792787525905167, + "tokens_seen": 700317696 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978836509528586, + "loss": 2.8797, + "theoretical_loss": 3.779242847260173, + "tokens_seen": 700383232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039787362086258777, + "loss": 2.7285, + "theoretical_loss": 3.7792069462300146, + "tokens_seen": 700448768 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397863590772317, + "loss": 2.6344, + "theoretical_loss": 3.779171049499123, + "tokens_seen": 700514304 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039785356068204613, + "loss": 2.7949, + "theoretical_loss": 3.779135157066581, + "tokens_seen": 700579840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039784353059177537, + "loss": 2.7319, + "theoretical_loss": 3.779099268931473, + "tokens_seen": 700645376 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978335005015045, + "loss": 2.836, + "theoretical_loss": 3.779063385092882, + "tokens_seen": 700710912 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039782347041123373, + "loss": 2.7745, + "theoretical_loss": 3.779027505549891, + "tokens_seen": 700776448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978134403209629, + "loss": 2.8877, + "theoretical_loss": 3.778991630301586, + "tokens_seen": 700841984 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003978034102306921, + "loss": 2.7129, + "theoretical_loss": 3.77895575934705, + "tokens_seen": 700907520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977933801404213, + "loss": 2.9092, + "theoretical_loss": 3.7789198926853684, + "tokens_seen": 700973056 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039778335005015046, + "loss": 2.7402, + "theoretical_loss": 3.7788840303156253, + "tokens_seen": 701038592 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039777331995987964, + "loss": 2.8797, + "theoretical_loss": 3.7788481722369065, + "tokens_seen": 701104128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039776328986960887, + "loss": 2.8667, + "theoretical_loss": 3.7788123184482982, + "tokens_seen": 701169664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 395841, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9108428955078125, + "objective/train/theoretical_loss": 3.7787764689488847, + "objective/train/tokens_used": 721695200, + "theoretical_loss": 3.7787764689488847, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397753259779338, + "loss": 2.8756, + "theoretical_loss": 3.7787764689488847, + "tokens_seen": 701235200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039774322968906723, + "loss": 2.907, + "theoretical_loss": 3.778740623737754, + "tokens_seen": 701300736 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977331995987964, + "loss": 2.5534, + "theoretical_loss": 3.778704782813991, + "tokens_seen": 701366272 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977231695085256, + "loss": 2.6829, + "theoretical_loss": 3.7786689461766825, + "tokens_seen": 701431808 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003977131394182548, + "loss": 2.5951, + "theoretical_loss": 3.778633113824916, + "tokens_seen": 701497344 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039770310932798396, + "loss": 2.9051, + "theoretical_loss": 3.778597285757778, + "tokens_seen": 701562880 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039769307923771314, + "loss": 2.7553, + "theoretical_loss": 3.7785614619743573, + "tokens_seen": 701628416 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976830491474424, + "loss": 2.5643, + "theoretical_loss": 3.77852564247374, + "tokens_seen": 701693952 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976730190571715, + "loss": 2.9458, + "theoretical_loss": 3.778489827255015, + "tokens_seen": 701759488 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039766298896690074, + "loss": 2.8461, + "theoretical_loss": 3.778454016317271, + "tokens_seen": 701825024 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039765295887662987, + "loss": 2.7984, + "theoretical_loss": 3.7784182096595957, + "tokens_seen": 701890560 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976429287863591, + "loss": 2.8552, + "theoretical_loss": 3.7783824072810788, + "tokens_seen": 701956096 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976328986960883, + "loss": 2.9651, + "theoretical_loss": 3.7783466091808093, + "tokens_seen": 702021632 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039762286860581746, + "loss": 2.8818, + "theoretical_loss": 3.7783108153578766, + "tokens_seen": 702087168 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039761283851554664, + "loss": 2.8327, + "theoretical_loss": 3.7782750258113698, + "tokens_seen": 702152704 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003976028084252759, + "loss": 2.5367, + "theoretical_loss": 3.7782392405403797, + "tokens_seen": 702218240 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397592778335005, + "loss": 2.8229, + "theoretical_loss": 3.7782034595439966, + "tokens_seen": 702283776 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039758274824473424, + "loss": 2.5208, + "theoretical_loss": 3.778167682821311, + "tokens_seen": 702349312 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039757271815446337, + "loss": 2.695, + "theoretical_loss": 3.778131910371413, + "tokens_seen": 702414848 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975626880641926, + "loss": 2.6583, + "theoretical_loss": 3.7780961421933945, + "tokens_seen": 702480384 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975526579739218, + "loss": 2.774, + "theoretical_loss": 3.7780603782863467, + "tokens_seen": 702545920 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039754262788365097, + "loss": 3.0157, + "theoretical_loss": 3.778024618649361, + "tokens_seen": 702611456 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039753259779338015, + "loss": 2.8291, + "theoretical_loss": 3.7779888632815295, + "tokens_seen": 702676992 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039752256770310933, + "loss": 2.859, + "theoretical_loss": 3.7779531121819447, + "tokens_seen": 702742528 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003975125376128385, + "loss": 2.8686, + "theoretical_loss": 3.7779173653496994, + "tokens_seen": 702808064 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 397024, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.391489028930664, + "objective/train/theoretical_loss": 3.7778816227838856, + "objective/train/tokens_used": 723333600, + "theoretical_loss": 3.7778816227838856, + "tokens_seen": 702873600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039750250752256774, + "loss": 2.806, + "theoretical_loss": 3.7778816227838856, + "tokens_seen": 702873600 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039749247743229687, + "loss": 2.8732, + "theoretical_loss": 3.7778458844835963, + "tokens_seen": 702939136 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974824473420261, + "loss": 2.7415, + "theoretical_loss": 3.777810150447926, + "tokens_seen": 703004672 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039747241725175523, + "loss": 2.728, + "theoretical_loss": 3.777774420675967, + "tokens_seen": 703070208 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039746238716148447, + "loss": 2.6557, + "theoretical_loss": 3.7777386951668133, + "tokens_seen": 703135744 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039745235707121365, + "loss": 2.6884, + "theoretical_loss": 3.7777029739195602, + "tokens_seen": 703201280 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039744232698094283, + "loss": 2.8055, + "theoretical_loss": 3.7776672569333014, + "tokens_seen": 703266816 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397432296890672, + "loss": 2.6844, + "theoretical_loss": 3.7776315442071313, + "tokens_seen": 703332352 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039742226680040125, + "loss": 2.7397, + "theoretical_loss": 3.7775958357401453, + "tokens_seen": 703397888 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974122367101304, + "loss": 2.8919, + "theoretical_loss": 3.7775601315314393, + "tokens_seen": 703463424 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003974022066198596, + "loss": 2.5523, + "theoretical_loss": 3.7775244315801078, + "tokens_seen": 703528960 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039739217652958874, + "loss": 2.6017, + "theoretical_loss": 3.777488735885247, + "tokens_seen": 703594496 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039738214643931797, + "loss": 2.8401, + "theoretical_loss": 3.777453044445953, + "tokens_seen": 703660032 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039737211634904715, + "loss": 2.7133, + "theoretical_loss": 3.7774173572613226, + "tokens_seen": 703725568 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039736208625877633, + "loss": 2.6392, + "theoretical_loss": 3.777381674330452, + "tokens_seen": 703791104 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973520561685055, + "loss": 2.8618, + "theoretical_loss": 3.7773459956524382, + "tokens_seen": 703856640 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973420260782347, + "loss": 2.7136, + "theoretical_loss": 3.7773103212263783, + "tokens_seen": 703922176 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973319959879639, + "loss": 2.7602, + "theoretical_loss": 3.7772746510513704, + "tokens_seen": 703987712 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973219658976931, + "loss": 2.7088, + "theoretical_loss": 3.7772389851265116, + "tokens_seen": 704053248 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039731193580742224, + "loss": 2.6738, + "theoretical_loss": 3.7772033234509, + "tokens_seen": 704118784 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003973019057171515, + "loss": 2.5981, + "theoretical_loss": 3.777167666023634, + "tokens_seen": 704184320 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972918756268806, + "loss": 2.9311, + "theoretical_loss": 3.777132012843812, + "tokens_seen": 704249856 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039728184553660984, + "loss": 2.7248, + "theoretical_loss": 3.777096363910534, + "tokens_seen": 704315392 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397271815446339, + "loss": 2.8256, + "theoretical_loss": 3.7770607192228978, + "tokens_seen": 704380928 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972617853560682, + "loss": 2.8975, + "theoretical_loss": 3.777025078780003, + "tokens_seen": 704446464 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 397723, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0135793685913086, + "objective/train/theoretical_loss": 3.77698944258095, + "objective/train/tokens_used": 724972000, + "theoretical_loss": 3.77698944258095, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972517552657974, + "loss": 2.7687, + "theoretical_loss": 3.77698944258095, + "tokens_seen": 704512000 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972417251755266, + "loss": 2.8214, + "theoretical_loss": 3.776953810624838, + "tokens_seen": 704577536 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039723169508525574, + "loss": 2.4458, + "theoretical_loss": 3.7769181829107676, + "tokens_seen": 704643072 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397221664994985, + "loss": 3.0101, + "theoretical_loss": 3.7768825594378397, + "tokens_seen": 704708608 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003972116349047141, + "loss": 2.8727, + "theoretical_loss": 3.7768469402051545, + "tokens_seen": 704774144 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039720160481444334, + "loss": 2.7502, + "theoretical_loss": 3.7768113252118134, + "tokens_seen": 704839680 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971915747241725, + "loss": 2.8048, + "theoretical_loss": 3.7767757144569174, + "tokens_seen": 704905216 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971815446339017, + "loss": 2.6821, + "theoretical_loss": 3.7767401079395677, + "tokens_seen": 704970752 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971715145436309, + "loss": 2.7107, + "theoretical_loss": 3.7767045056588677, + "tokens_seen": 705036288 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039716148445336007, + "loss": 2.9334, + "theoretical_loss": 3.776668907613918, + "tokens_seen": 705101824 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039715145436308925, + "loss": 2.7057, + "theoretical_loss": 3.7766333138038224, + "tokens_seen": 705167360 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971414242728185, + "loss": 2.6521, + "theoretical_loss": 3.776597724227682, + "tokens_seen": 705232896 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039713139418254766, + "loss": 2.5163, + "theoretical_loss": 3.776562138884601, + "tokens_seen": 705298432 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039712136409227684, + "loss": 2.9374, + "theoretical_loss": 3.7765265577736824, + "tokens_seen": 705363968 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971113340020061, + "loss": 2.6889, + "theoretical_loss": 3.7764909808940295, + "tokens_seen": 705429504 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003971013039117352, + "loss": 2.7154, + "theoretical_loss": 3.776455408244746, + "tokens_seen": 705495040 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039709127382146444, + "loss": 2.7799, + "theoretical_loss": 3.7764198398249365, + "tokens_seen": 705560576 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039708124373119357, + "loss": 2.8133, + "theoretical_loss": 3.7763842756337054, + "tokens_seen": 705626112 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970712136409228, + "loss": 2.6184, + "theoretical_loss": 3.776348715670156, + "tokens_seen": 705691648 + }, + { + "epoch": 0.21, + "learning_rate": 0.000397061183550652, + "loss": 2.6634, + "theoretical_loss": 3.7763131599333946, + "tokens_seen": 705757184 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039705115346038117, + "loss": 2.6501, + "theoretical_loss": 3.776277608422526, + "tokens_seen": 705822720 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039704112337011035, + "loss": 2.7362, + "theoretical_loss": 3.7762420611366556, + "tokens_seen": 705888256 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039703109327983953, + "loss": 2.9123, + "theoretical_loss": 3.776206518074889, + "tokens_seen": 705953792 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003970210631895687, + "loss": 2.6798, + "theoretical_loss": 3.776170979236332, + "tokens_seen": 706019328 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039701103309929794, + "loss": 2.7245, + "theoretical_loss": 3.7761354446200914, + "tokens_seen": 706084864 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 398840, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.317671775817871, + "objective/train/theoretical_loss": 3.776099914225273, + "objective/train/tokens_used": 726610400, + "theoretical_loss": 3.776099914225273, + "tokens_seen": 706150400 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039700100300902707, + "loss": 2.7278, + "theoretical_loss": 3.776099914225273, + "tokens_seen": 706150400 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969909729187563, + "loss": 2.8361, + "theoretical_loss": 3.7760643880509845, + "tokens_seen": 706215936 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039698094282848543, + "loss": 2.8228, + "theoretical_loss": 3.776028866096332, + "tokens_seen": 706281472 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039697091273821467, + "loss": 2.8389, + "theoretical_loss": 3.7759933483604238, + "tokens_seen": 706347008 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039696088264794385, + "loss": 2.7686, + "theoretical_loss": 3.775957834842367, + "tokens_seen": 706412544 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039695085255767303, + "loss": 2.8636, + "theoretical_loss": 3.775922325541269, + "tokens_seen": 706478080 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969408224674022, + "loss": 2.8, + "theoretical_loss": 3.775886820456239, + "tokens_seen": 706543616 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039693079237713145, + "loss": 2.8553, + "theoretical_loss": 3.7758513195863843, + "tokens_seen": 706609152 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969207622868606, + "loss": 2.7078, + "theoretical_loss": 3.7758158229308143, + "tokens_seen": 706674688 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003969107321965898, + "loss": 2.6724, + "theoretical_loss": 3.775780330488638, + "tokens_seen": 706740224 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039690070210631894, + "loss": 2.797, + "theoretical_loss": 3.7757448422589643, + "tokens_seen": 706805760 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039689067201604817, + "loss": 2.8585, + "theoretical_loss": 3.7757093582409036, + "tokens_seen": 706871296 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039688064192577735, + "loss": 2.5493, + "theoretical_loss": 3.7756738784335644, + "tokens_seen": 706936832 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039687061183550653, + "loss": 2.6593, + "theoretical_loss": 3.775638402836057, + "tokens_seen": 707002368 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968605817452357, + "loss": 2.8549, + "theoretical_loss": 3.7756029314474926, + "tokens_seen": 707067904 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968505516549649, + "loss": 2.6769, + "theoretical_loss": 3.775567464266981, + "tokens_seen": 707133440 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968405215646941, + "loss": 2.8655, + "theoretical_loss": 3.7755320012936338, + "tokens_seen": 707198976 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968304914744233, + "loss": 2.7365, + "theoretical_loss": 3.775496542526561, + "tokens_seen": 707264512 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039682046138415244, + "loss": 3.1246, + "theoretical_loss": 3.7754610879648745, + "tokens_seen": 707330048 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968104312938817, + "loss": 2.9093, + "theoretical_loss": 3.775425637607687, + "tokens_seen": 707395584 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003968004012036108, + "loss": 2.7906, + "theoretical_loss": 3.775390191454109, + "tokens_seen": 707461120 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039679037111334004, + "loss": 2.6881, + "theoretical_loss": 3.7753547495032533, + "tokens_seen": 707526656 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967803410230692, + "loss": 2.6145, + "theoretical_loss": 3.775319311754232, + "tokens_seen": 707592192 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967703109327984, + "loss": 2.8557, + "theoretical_loss": 3.775283878206159, + "tokens_seen": 707657728 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967602808425276, + "loss": 2.8326, + "theoretical_loss": 3.7752484488581457, + "tokens_seen": 707723264 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 399432, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6968586444854736, + "objective/train/theoretical_loss": 3.7752130237093064, + "objective/train/tokens_used": 728248800, + "theoretical_loss": 3.7752130237093064, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967502507522568, + "loss": 2.6011, + "theoretical_loss": 3.7752130237093064, + "tokens_seen": 707788800 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039674022066198594, + "loss": 2.7763, + "theoretical_loss": 3.775177602758755, + "tokens_seen": 707854336 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967301905717152, + "loss": 2.9726, + "theoretical_loss": 3.7751421860056045, + "tokens_seen": 707919872 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967201604814443, + "loss": 2.7084, + "theoretical_loss": 3.775106773448969, + "tokens_seen": 707985408 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039671013039117354, + "loss": 2.8755, + "theoretical_loss": 3.7750713650879635, + "tokens_seen": 708050944 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003967001003009027, + "loss": 2.7319, + "theoretical_loss": 3.775035960921702, + "tokens_seen": 708116480 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966900702106319, + "loss": 2.816, + "theoretical_loss": 3.7750005609493003, + "tokens_seen": 708182016 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966800401203611, + "loss": 2.6943, + "theoretical_loss": 3.774965165169873, + "tokens_seen": 708247552 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039667001003009027, + "loss": 2.7305, + "theoretical_loss": 3.774929773582535, + "tokens_seen": 708313088 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039665997993981945, + "loss": 2.8117, + "theoretical_loss": 3.7748943861864026, + "tokens_seen": 708378624 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966499498495487, + "loss": 2.7627, + "theoretical_loss": 3.774859002980592, + "tokens_seen": 708444160 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966399197592778, + "loss": 2.8163, + "theoretical_loss": 3.774823623964219, + "tokens_seen": 708509696 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039662988966900704, + "loss": 2.7808, + "theoretical_loss": 3.7747882491364004, + "tokens_seen": 708575232 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039661985957873617, + "loss": 2.7834, + "theoretical_loss": 3.774752878496253, + "tokens_seen": 708640768 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003966098294884654, + "loss": 2.867, + "theoretical_loss": 3.7747175120428933, + "tokens_seen": 708706304 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965997993981946, + "loss": 2.8166, + "theoretical_loss": 3.774682149775439, + "tokens_seen": 708771840 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039658976930792377, + "loss": 2.8267, + "theoretical_loss": 3.7746467916930078, + "tokens_seen": 708837376 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039657973921765295, + "loss": 2.5516, + "theoretical_loss": 3.774611437794718, + "tokens_seen": 708902912 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965697091273822, + "loss": 2.8087, + "theoretical_loss": 3.7745760880796864, + "tokens_seen": 708968448 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965596790371113, + "loss": 2.8472, + "theoretical_loss": 3.7745407425470328, + "tokens_seen": 709033984 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039654964894684055, + "loss": 2.6325, + "theoretical_loss": 3.774505401195875, + "tokens_seen": 709099520 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965396188565697, + "loss": 2.6698, + "theoretical_loss": 3.774470064025332, + "tokens_seen": 709165056 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965295887662989, + "loss": 2.4007, + "theoretical_loss": 3.774434731034524, + "tokens_seen": 709230592 + }, + { + "epoch": 0.21, + "learning_rate": 0.0003965195586760281, + "loss": 2.8729, + "theoretical_loss": 3.7743994022225684, + "tokens_seen": 709296128 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039650952858575727, + "loss": 2.6295, + "theoretical_loss": 3.774364077588587, + "tokens_seen": 709361664 + }, + { + "epoch": 0.21, + "objective/train/docs_used": 400468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9617061614990234, + "objective/train/theoretical_loss": 3.7743287571316984, + "objective/train/tokens_used": 729887200, + "theoretical_loss": 3.7743287571316984, + "tokens_seen": 709427200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039649949849548645, + "loss": 2.8305, + "theoretical_loss": 3.7743287571316984, + "tokens_seen": 709427200 + }, + { + "epoch": 0.21, + "learning_rate": 0.00039648946840521563, + "loss": 2.9433, + "theoretical_loss": 3.774293440851024, + "tokens_seen": 709492736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964794383149448, + "loss": 2.7475, + "theoretical_loss": 3.774258128745683, + "tokens_seen": 709558272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039646940822467405, + "loss": 2.7497, + "theoretical_loss": 3.7742228208147974, + "tokens_seen": 709623808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964593781344032, + "loss": 2.5815, + "theoretical_loss": 3.7741875170574875, + "tokens_seen": 709689344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964493480441324, + "loss": 2.7236, + "theoretical_loss": 3.774152217472875, + "tokens_seen": 709754880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964393179538616, + "loss": 2.6442, + "theoretical_loss": 3.774116922060081, + "tokens_seen": 709820416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003964292878635908, + "loss": 2.8988, + "theoretical_loss": 3.7740816308182286, + "tokens_seen": 709885952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039641925777331996, + "loss": 2.9272, + "theoretical_loss": 3.7740463437464387, + "tokens_seen": 709951488 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039640922768304914, + "loss": 2.836, + "theoretical_loss": 3.7740110608438338, + "tokens_seen": 710017024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963991975927783, + "loss": 2.9205, + "theoretical_loss": 3.7739757821095363, + "tokens_seen": 710082560 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039638916750250755, + "loss": 2.7784, + "theoretical_loss": 3.7739405075426706, + "tokens_seen": 710148096 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039637913741223673, + "loss": 2.7215, + "theoretical_loss": 3.7739052371423583, + "tokens_seen": 710213632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963691073219659, + "loss": 2.76, + "theoretical_loss": 3.7738699709077235, + "tokens_seen": 710279168 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963590772316951, + "loss": 2.7615, + "theoretical_loss": 3.77383470883789, + "tokens_seen": 710344704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963490471414243, + "loss": 2.6549, + "theoretical_loss": 3.773799450931982, + "tokens_seen": 710410240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963390170511535, + "loss": 2.7139, + "theoretical_loss": 3.7737641971891227, + "tokens_seen": 710475776 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039632898696088264, + "loss": 2.8427, + "theoretical_loss": 3.7737289476084372, + "tokens_seen": 710541312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003963189568706119, + "loss": 2.8954, + "theoretical_loss": 3.773693702189051, + "tokens_seen": 710606848 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396308926780341, + "loss": 2.6769, + "theoretical_loss": 3.773658460930088, + "tokens_seen": 710672384 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039629889669007024, + "loss": 2.7509, + "theoretical_loss": 3.7736232238306746, + "tokens_seen": 710737920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962888665997994, + "loss": 2.7514, + "theoretical_loss": 3.7735879908899355, + "tokens_seen": 710803456 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962788365095286, + "loss": 2.7883, + "theoretical_loss": 3.7735527621069966, + "tokens_seen": 710868992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962688064192578, + "loss": 2.935, + "theoretical_loss": 3.773517537480984, + "tokens_seen": 710934528 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396258776328987, + "loss": 2.6263, + "theoretical_loss": 3.773482317011025, + "tokens_seen": 711000064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 401088, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.054889440536499, + "objective/train/theoretical_loss": 3.773447100696245, + "objective/train/tokens_used": 731525600, + "theoretical_loss": 3.773447100696245, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039624874623871614, + "loss": 2.9868, + "theoretical_loss": 3.773447100696245, + "tokens_seen": 711065600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962387161484454, + "loss": 2.7579, + "theoretical_loss": 3.7734118885357715, + "tokens_seen": 711131136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962286860581745, + "loss": 2.8857, + "theoretical_loss": 3.7733766805287314, + "tokens_seen": 711196672 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039621865596790374, + "loss": 2.9905, + "theoretical_loss": 3.7733414766742523, + "tokens_seen": 711262208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003962086258776329, + "loss": 2.584, + "theoretical_loss": 3.7733062769714616, + "tokens_seen": 711327744 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961985957873621, + "loss": 2.625, + "theoretical_loss": 3.7732710814194883, + "tokens_seen": 711393280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961885656970913, + "loss": 2.8167, + "theoretical_loss": 3.7732358900174585, + "tokens_seen": 711458816 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039617853560682047, + "loss": 2.9579, + "theoretical_loss": 3.7732007027645027, + "tokens_seen": 711524352 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039616850551654965, + "loss": 2.876, + "theoretical_loss": 3.7731655196597496, + "tokens_seen": 711589888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961584754262789, + "loss": 2.757, + "theoretical_loss": 3.7731303407023264, + "tokens_seen": 711655424 + }, + { + "epoch": 0.22, + "learning_rate": 0.000396148445336008, + "loss": 2.8455, + "theoretical_loss": 3.773095165891364, + "tokens_seen": 711720960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039613841524573724, + "loss": 2.6882, + "theoretical_loss": 3.7730599952259913, + "tokens_seen": 711786496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039612838515546637, + "loss": 2.7801, + "theoretical_loss": 3.7730248287053376, + "tokens_seen": 711852032 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961183550651956, + "loss": 2.8345, + "theoretical_loss": 3.772989666328534, + "tokens_seen": 711917568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003961083249749248, + "loss": 2.8088, + "theoretical_loss": 3.7729545080947102, + "tokens_seen": 711983104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039609829488465397, + "loss": 2.4923, + "theoretical_loss": 3.772919354002997, + "tokens_seen": 712048640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039608826479438315, + "loss": 2.6372, + "theoretical_loss": 3.772884204052525, + "tokens_seen": 712114176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960782347041124, + "loss": 2.7558, + "theoretical_loss": 3.7728490582424254, + "tokens_seen": 712179712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960682046138415, + "loss": 2.7638, + "theoretical_loss": 3.7728139165718297, + "tokens_seen": 712245248 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039605817452357075, + "loss": 2.875, + "theoretical_loss": 3.7727787790398697, + "tokens_seen": 712310784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960481444332999, + "loss": 2.9478, + "theoretical_loss": 3.7727436456456767, + "tokens_seen": 712376320 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960381143430291, + "loss": 2.7149, + "theoretical_loss": 3.772708516388383, + "tokens_seen": 712441856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003960280842527583, + "loss": 2.8476, + "theoretical_loss": 3.772673391267121, + "tokens_seen": 712507392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039601805416248747, + "loss": 2.7814, + "theoretical_loss": 3.772638270281024, + "tokens_seen": 712572928 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039600802407221665, + "loss": 2.7704, + "theoretical_loss": 3.7726031534292246, + "tokens_seen": 712638464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 402378, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6918859481811523, + "objective/train/theoretical_loss": 3.7725680407108557, + "objective/train/tokens_used": 733164000, + "theoretical_loss": 3.7725680407108557, + "tokens_seen": 712704000 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039599799398194583, + "loss": 2.7392, + "theoretical_loss": 3.7725680407108557, + "tokens_seen": 712704000 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395987963891675, + "loss": 2.6279, + "theoretical_loss": 3.7725329321250505, + "tokens_seen": 712769536 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039597793380140425, + "loss": 2.7209, + "theoretical_loss": 3.7724978276709433, + "tokens_seen": 712835072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959679037111334, + "loss": 2.7042, + "theoretical_loss": 3.772462727347668, + "tokens_seen": 712900608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959578736208626, + "loss": 2.6791, + "theoretical_loss": 3.7724276311543585, + "tokens_seen": 712966144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959478435305918, + "loss": 2.8125, + "theoretical_loss": 3.7723925390901503, + "tokens_seen": 713031680 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395937813440321, + "loss": 2.7226, + "theoretical_loss": 3.7723574511541766, + "tokens_seen": 713097216 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039592778335005016, + "loss": 2.6501, + "theoretical_loss": 3.7723223673455735, + "tokens_seen": 713162752 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039591775325977934, + "loss": 2.8047, + "theoretical_loss": 3.772287287663476, + "tokens_seen": 713228288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003959077231695085, + "loss": 2.7207, + "theoretical_loss": 3.77225221210702, + "tokens_seen": 713293824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039589769307923775, + "loss": 2.7146, + "theoretical_loss": 3.7722171406753406, + "tokens_seen": 713359360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958876629889669, + "loss": 2.713, + "theoretical_loss": 3.772182073367574, + "tokens_seen": 713424896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958776328986961, + "loss": 2.8498, + "theoretical_loss": 3.7721470101828576, + "tokens_seen": 713490432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039586760280842524, + "loss": 2.3899, + "theoretical_loss": 3.7721119511203263, + "tokens_seen": 713555968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958575727181545, + "loss": 2.5299, + "theoretical_loss": 3.7720768961791182, + "tokens_seen": 713621504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039584754262788366, + "loss": 2.6941, + "theoretical_loss": 3.7720418453583697, + "tokens_seen": 713687040 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039583751253761284, + "loss": 2.6472, + "theoretical_loss": 3.7720067986572188, + "tokens_seen": 713752576 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395827482447342, + "loss": 2.7839, + "theoretical_loss": 3.7719717560748025, + "tokens_seen": 713818112 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958174523570712, + "loss": 2.7576, + "theoretical_loss": 3.771936717610259, + "tokens_seen": 713883648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003958074222668004, + "loss": 2.722, + "theoretical_loss": 3.7719016832627266, + "tokens_seen": 713949184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957973921765296, + "loss": 2.44, + "theoretical_loss": 3.7718666530313434, + "tokens_seen": 714014720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039578736208625875, + "loss": 2.7879, + "theoretical_loss": 3.7718316269152483, + "tokens_seen": 714080256 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395777331995988, + "loss": 3.0367, + "theoretical_loss": 3.77179660491358, + "tokens_seen": 714145792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039576730190571716, + "loss": 2.8812, + "theoretical_loss": 3.7717615870254777, + "tokens_seen": 714211328 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039575727181544634, + "loss": 2.9063, + "theoretical_loss": 3.7717265732500804, + "tokens_seen": 714276864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 402960, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7978687286376953, + "objective/train/theoretical_loss": 3.771691563586529, + "objective/train/tokens_used": 734802400, + "theoretical_loss": 3.771691563586529, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957472417251755, + "loss": 2.6231, + "theoretical_loss": 3.771691563586529, + "tokens_seen": 714342400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957372116349047, + "loss": 2.7291, + "theoretical_loss": 3.7716565580339623, + "tokens_seen": 714407936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957271815446339, + "loss": 2.7152, + "theoretical_loss": 3.771621556591522, + "tokens_seen": 714473472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003957171514543631, + "loss": 2.7035, + "theoretical_loss": 3.7715865592583464, + "tokens_seen": 714539008 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039570712136409225, + "loss": 2.6655, + "theoretical_loss": 3.771551566033578, + "tokens_seen": 714604544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956970912738215, + "loss": 2.6651, + "theoretical_loss": 3.771516576916357, + "tokens_seen": 714670080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956870611835506, + "loss": 2.6674, + "theoretical_loss": 3.7714815919058244, + "tokens_seen": 714735616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039567703109327985, + "loss": 2.7265, + "theoretical_loss": 3.771446611001123, + "tokens_seen": 714801152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039566700100300903, + "loss": 2.6466, + "theoretical_loss": 3.7714116342013932, + "tokens_seen": 714866688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956569709127382, + "loss": 2.8245, + "theoretical_loss": 3.7713766615057778, + "tokens_seen": 714932224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956469408224674, + "loss": 2.7489, + "theoretical_loss": 3.7713416929134187, + "tokens_seen": 714997760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039563691073219657, + "loss": 2.6609, + "theoretical_loss": 3.7713067284234585, + "tokens_seen": 715063296 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003956268806419258, + "loss": 2.6133, + "theoretical_loss": 3.7712717680350405, + "tokens_seen": 715128832 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395616850551655, + "loss": 2.7701, + "theoretical_loss": 3.7712368117473067, + "tokens_seen": 715194368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039560682046138417, + "loss": 2.5693, + "theoretical_loss": 3.771201859559401, + "tokens_seen": 715259904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039559679037111335, + "loss": 2.9091, + "theoretical_loss": 3.771166911470468, + "tokens_seen": 715325440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955867602808426, + "loss": 2.6383, + "theoretical_loss": 3.7711319674796497, + "tokens_seen": 715390976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955767301905717, + "loss": 2.8301, + "theoretical_loss": 3.771097027586091, + "tokens_seen": 715456512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039556670010030095, + "loss": 2.8646, + "theoretical_loss": 3.7710620917889366, + "tokens_seen": 715522048 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955566700100301, + "loss": 2.664, + "theoretical_loss": 3.77102716008733, + "tokens_seen": 715587584 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955466399197593, + "loss": 2.5999, + "theoretical_loss": 3.7709922324804177, + "tokens_seen": 715653120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003955366098294885, + "loss": 2.9666, + "theoretical_loss": 3.7709573089673443, + "tokens_seen": 715718656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039552657973921767, + "loss": 2.6747, + "theoretical_loss": 3.7709223895472537, + "tokens_seen": 715784192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039551654964894685, + "loss": 2.4762, + "theoretical_loss": 3.7708874742192933, + "tokens_seen": 715849728 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039550651955867603, + "loss": 2.8351, + "theoretical_loss": 3.7708525629826077, + "tokens_seen": 715915264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 403589, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7969202995300293, + "objective/train/theoretical_loss": 3.7708176558363444, + "objective/train/tokens_used": 736440800, + "theoretical_loss": 3.7708176558363444, + "tokens_seen": 715980800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954964894684052, + "loss": 2.5087, + "theoretical_loss": 3.7708176558363444, + "tokens_seen": 715980800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039548645937813445, + "loss": 2.7577, + "theoretical_loss": 3.770782752779649, + "tokens_seen": 716046336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954764292878636, + "loss": 2.8457, + "theoretical_loss": 3.7707478538116677, + "tokens_seen": 716111872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954663991975928, + "loss": 2.5495, + "theoretical_loss": 3.7707129589315485, + "tokens_seen": 716177408 + }, + { + "epoch": 0.22, + "learning_rate": 0.000395456369107322, + "loss": 2.7968, + "theoretical_loss": 3.7706780681384373, + "tokens_seen": 716242944 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954463390170512, + "loss": 2.5126, + "theoretical_loss": 3.770643181431483, + "tokens_seen": 716308480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039543630892678036, + "loss": 2.7764, + "theoretical_loss": 3.770608298809832, + "tokens_seen": 716374016 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039542627883650954, + "loss": 2.7939, + "theoretical_loss": 3.770573420272633, + "tokens_seen": 716439552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003954162487462387, + "loss": 2.6007, + "theoretical_loss": 3.7705385458190337, + "tokens_seen": 716505088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039540621865596795, + "loss": 2.7519, + "theoretical_loss": 3.7705036754481824, + "tokens_seen": 716570624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953961885656971, + "loss": 2.9651, + "theoretical_loss": 3.7704688091592287, + "tokens_seen": 716636160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953861584754263, + "loss": 2.511, + "theoretical_loss": 3.7704339469513206, + "tokens_seen": 716701696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039537612838515544, + "loss": 2.6907, + "theoretical_loss": 3.770399088823608, + "tokens_seen": 716767232 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953660982948847, + "loss": 2.5684, + "theoretical_loss": 3.77036423477524, + "tokens_seen": 716832768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039535606820461386, + "loss": 2.6212, + "theoretical_loss": 3.7703293848053665, + "tokens_seen": 716898304 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039534603811434304, + "loss": 2.618, + "theoretical_loss": 3.770294538913137, + "tokens_seen": 716963840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953360080240722, + "loss": 2.6721, + "theoretical_loss": 3.7702596970977025, + "tokens_seen": 717029376 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953259779338014, + "loss": 2.7419, + "theoretical_loss": 3.770224859358213, + "tokens_seen": 717094912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953159478435306, + "loss": 2.7789, + "theoretical_loss": 3.770190025693819, + "tokens_seen": 717160448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003953059177532598, + "loss": 2.6623, + "theoretical_loss": 3.7701551961036723, + "tokens_seen": 717225984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039529588766298895, + "loss": 2.9295, + "theoretical_loss": 3.7701203705869233, + "tokens_seen": 717291520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952858575727182, + "loss": 2.7038, + "theoretical_loss": 3.770085549142724, + "tokens_seen": 717357056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039527582748244736, + "loss": 2.759, + "theoretical_loss": 3.770050731770226, + "tokens_seen": 717422592 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039526579739217654, + "loss": 2.4058, + "theoretical_loss": 3.770015918468582, + "tokens_seen": 717488128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952557673019057, + "loss": 2.7957, + "theoretical_loss": 3.7699811092369426, + "tokens_seen": 717553664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 404678, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7875523567199707, + "objective/train/theoretical_loss": 3.7699463040744616, + "objective/train/tokens_used": 738079200, + "theoretical_loss": 3.7699463040744616, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952457372116349, + "loss": 2.6614, + "theoretical_loss": 3.7699463040744616, + "tokens_seen": 717619200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952357071213641, + "loss": 2.672, + "theoretical_loss": 3.7699115029802917, + "tokens_seen": 717684736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952256770310933, + "loss": 2.7551, + "theoretical_loss": 3.769876705953586, + "tokens_seen": 717750272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039521564694082245, + "loss": 2.6962, + "theoretical_loss": 3.769841912993497, + "tokens_seen": 717815808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003952056168505517, + "loss": 2.7611, + "theoretical_loss": 3.769807124099179, + "tokens_seen": 717881344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951955867602808, + "loss": 2.8147, + "theoretical_loss": 3.7697723392697853, + "tokens_seen": 717946880 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039518555667001005, + "loss": 2.7761, + "theoretical_loss": 3.7697375585044703, + "tokens_seen": 718012416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039517552657973923, + "loss": 2.8367, + "theoretical_loss": 3.769702781802388, + "tokens_seen": 718077952 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951654964894684, + "loss": 2.6044, + "theoretical_loss": 3.769668009162693, + "tokens_seen": 718143488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951554663991976, + "loss": 2.5959, + "theoretical_loss": 3.7696332405845405, + "tokens_seen": 718209024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039514543630892677, + "loss": 2.517, + "theoretical_loss": 3.7695984760670855, + "tokens_seen": 718274560 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039513540621865595, + "loss": 2.7682, + "theoretical_loss": 3.7695637156094826, + "tokens_seen": 718340096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951253761283852, + "loss": 2.4989, + "theoretical_loss": 3.769528959210888, + "tokens_seen": 718405632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003951153460381143, + "loss": 2.647, + "theoretical_loss": 3.7694942068704576, + "tokens_seen": 718471168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039510531594784355, + "loss": 2.735, + "theoretical_loss": 3.7694594585873467, + "tokens_seen": 718536704 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039509528585757273, + "loss": 2.6373, + "theoretical_loss": 3.7694247143607127, + "tokens_seen": 718602240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950852557673019, + "loss": 2.8542, + "theoretical_loss": 3.7693899741897106, + "tokens_seen": 718667776 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950752256770311, + "loss": 2.843, + "theoretical_loss": 3.7693552380734987, + "tokens_seen": 718733312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950651955867603, + "loss": 2.7613, + "theoretical_loss": 3.7693205060112343, + "tokens_seen": 718798848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039505516549648946, + "loss": 2.7541, + "theoretical_loss": 3.769285778002073, + "tokens_seen": 718864384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950451354062187, + "loss": 2.5525, + "theoretical_loss": 3.769251054045174, + "tokens_seen": 718929920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950351053159478, + "loss": 2.7406, + "theoretical_loss": 3.769216334139694, + "tokens_seen": 718995456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039502507522567705, + "loss": 2.6661, + "theoretical_loss": 3.769181618284792, + "tokens_seen": 719060992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950150451354062, + "loss": 2.4692, + "theoretical_loss": 3.7691469064796257, + "tokens_seen": 719126528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003950050150451354, + "loss": 2.5154, + "theoretical_loss": 3.769112198723354, + "tokens_seen": 719192064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 405859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0279910564422607, + "objective/train/theoretical_loss": 3.769077495015136, + "objective/train/tokens_used": 739717600, + "theoretical_loss": 3.769077495015136, + "tokens_seen": 719257600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949949849548646, + "loss": 2.6503, + "theoretical_loss": 3.769077495015136, + "tokens_seen": 719257600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949849548645938, + "loss": 2.8353, + "theoretical_loss": 3.76904279535413, + "tokens_seen": 719323136 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039497492477432296, + "loss": 2.6663, + "theoretical_loss": 3.7690080997394952, + "tokens_seen": 719388672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949648946840522, + "loss": 2.7235, + "theoretical_loss": 3.7689734081703925, + "tokens_seen": 719454208 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949548645937813, + "loss": 2.5129, + "theoretical_loss": 3.7689387206459806, + "tokens_seen": 719519744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039494483450351056, + "loss": 2.8694, + "theoretical_loss": 3.7689040371654206, + "tokens_seen": 719585280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949348044132397, + "loss": 2.801, + "theoretical_loss": 3.768869357727872, + "tokens_seen": 719650816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949247743229689, + "loss": 2.8021, + "theoretical_loss": 3.7688346823324954, + "tokens_seen": 719716352 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949147442326981, + "loss": 2.8085, + "theoretical_loss": 3.7688000109784516, + "tokens_seen": 719781888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003949047141424273, + "loss": 2.6279, + "theoretical_loss": 3.768765343664902, + "tokens_seen": 719847424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948946840521565, + "loss": 2.6426, + "theoretical_loss": 3.7687306803910086, + "tokens_seen": 719912960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039488465396188564, + "loss": 2.7365, + "theoretical_loss": 3.7686960211559315, + "tokens_seen": 719978496 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948746238716149, + "loss": 2.8506, + "theoretical_loss": 3.7686613659588337, + "tokens_seen": 720044032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039486459378134406, + "loss": 2.9218, + "theoretical_loss": 3.7686267147988772, + "tokens_seen": 720109568 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039485456369107324, + "loss": 2.7985, + "theoretical_loss": 3.7685920676752236, + "tokens_seen": 720175104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948445336008024, + "loss": 2.7395, + "theoretical_loss": 3.768557424587036, + "tokens_seen": 720240640 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948345035105316, + "loss": 2.9131, + "theoretical_loss": 3.7685227855334777, + "tokens_seen": 720306176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003948244734202608, + "loss": 2.7174, + "theoretical_loss": 3.7684881505137113, + "tokens_seen": 720371712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039481444332999, + "loss": 3.0072, + "theoretical_loss": 3.7684535195268998, + "tokens_seen": 720437248 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039480441323971915, + "loss": 2.7996, + "theoretical_loss": 3.7684188925722073, + "tokens_seen": 720502784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947943831494484, + "loss": 2.8799, + "theoretical_loss": 3.7683842696487977, + "tokens_seen": 720568320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039478435305917756, + "loss": 2.6279, + "theoretical_loss": 3.7683496507558347, + "tokens_seen": 720633856 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039477432296890674, + "loss": 3.06, + "theoretical_loss": 3.7683150358924826, + "tokens_seen": 720699392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947642928786359, + "loss": 3.0075, + "theoretical_loss": 3.7682804250579065, + "tokens_seen": 720764928 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947542627883651, + "loss": 2.9534, + "theoretical_loss": 3.768245818251271, + "tokens_seen": 720830464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 406138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.268052577972412, + "objective/train/theoretical_loss": 3.768211215471741, + "objective/train/tokens_used": 741356000, + "theoretical_loss": 3.768211215471741, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947442326980943, + "loss": 2.7847, + "theoretical_loss": 3.768211215471741, + "tokens_seen": 720896000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947342026078235, + "loss": 2.6443, + "theoretical_loss": 3.768176616718482, + "tokens_seen": 720961536 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039472417251755265, + "loss": 2.5376, + "theoretical_loss": 3.7681420219906596, + "tokens_seen": 721027072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003947141424272819, + "loss": 2.579, + "theoretical_loss": 3.76810743128744, + "tokens_seen": 721092608 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394704112337011, + "loss": 2.7805, + "theoretical_loss": 3.768072844607988, + "tokens_seen": 721158144 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039469408224674025, + "loss": 2.6879, + "theoretical_loss": 3.768038261951472, + "tokens_seen": 721223680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039468405215646943, + "loss": 2.6076, + "theoretical_loss": 3.7680036833170565, + "tokens_seen": 721289216 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946740220661986, + "loss": 2.699, + "theoretical_loss": 3.7679691087039098, + "tokens_seen": 721354752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946639919759278, + "loss": 2.7964, + "theoretical_loss": 3.7679345381111986, + "tokens_seen": 721420288 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039465396188565697, + "loss": 2.5689, + "theoretical_loss": 3.76789997153809, + "tokens_seen": 721485824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039464393179538615, + "loss": 2.7629, + "theoretical_loss": 3.767865408983751, + "tokens_seen": 721551360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946339017051154, + "loss": 2.6272, + "theoretical_loss": 3.7678308504473508, + "tokens_seen": 721616896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003946238716148445, + "loss": 2.581, + "theoretical_loss": 3.767796295928056, + "tokens_seen": 721682432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039461384152457375, + "loss": 2.7804, + "theoretical_loss": 3.767761745425036, + "tokens_seen": 721747968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039460381143430293, + "loss": 2.8805, + "theoretical_loss": 3.767727198937459, + "tokens_seen": 721813504 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945937813440321, + "loss": 2.8968, + "theoretical_loss": 3.767692656464494, + "tokens_seen": 721879040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945837512537613, + "loss": 2.6535, + "theoretical_loss": 3.76765811800531, + "tokens_seen": 721944576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945737211634905, + "loss": 2.8241, + "theoretical_loss": 3.767623583559076, + "tokens_seen": 722010112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039456369107321966, + "loss": 2.9034, + "theoretical_loss": 3.767589053124962, + "tokens_seen": 722075648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945536609829489, + "loss": 3.0686, + "theoretical_loss": 3.767554526702137, + "tokens_seen": 722141184 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394543630892678, + "loss": 2.8172, + "theoretical_loss": 3.7675200042897723, + "tokens_seen": 722206720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039453360080240725, + "loss": 2.8621, + "theoretical_loss": 3.767485485887037, + "tokens_seen": 722272256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945235707121364, + "loss": 2.8103, + "theoretical_loss": 3.7674509714931026, + "tokens_seen": 722337792 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945135406218656, + "loss": 2.893, + "theoretical_loss": 3.767416461107139, + "tokens_seen": 722403328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003945035105315948, + "loss": 2.8334, + "theoretical_loss": 3.767381954728318, + "tokens_seen": 722468864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 407490, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3906686305999756, + "objective/train/theoretical_loss": 3.7673474523558106, + "objective/train/tokens_used": 742994400, + "theoretical_loss": 3.7673474523558106, + "tokens_seen": 722534400 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394493480441324, + "loss": 2.661, + "theoretical_loss": 3.7673474523558106, + "tokens_seen": 722534400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039448345035105316, + "loss": 2.7209, + "theoretical_loss": 3.767312953988788, + "tokens_seen": 722599936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944734202607824, + "loss": 2.5782, + "theoretical_loss": 3.767278459626423, + "tokens_seen": 722665472 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944633901705115, + "loss": 2.589, + "theoretical_loss": 3.767243969267886, + "tokens_seen": 722731008 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039445336008024076, + "loss": 2.7244, + "theoretical_loss": 3.7672094829123512, + "tokens_seen": 722796544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944433299899699, + "loss": 2.6615, + "theoretical_loss": 3.7671750005589892, + "tokens_seen": 722862080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944332998996991, + "loss": 3.0289, + "theoretical_loss": 3.767140522206974, + "tokens_seen": 722927616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944232698094283, + "loss": 2.6406, + "theoretical_loss": 3.7671060478554783, + "tokens_seen": 722993152 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003944132397191575, + "loss": 2.746, + "theoretical_loss": 3.7670715775036756, + "tokens_seen": 723058688 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039440320962888666, + "loss": 2.9204, + "theoretical_loss": 3.7670371111507386, + "tokens_seen": 723124224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039439317953861584, + "loss": 2.5796, + "theoretical_loss": 3.7670026487958417, + "tokens_seen": 723189760 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394383149448345, + "loss": 2.6218, + "theoretical_loss": 3.766968190438159, + "tokens_seen": 723255296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039437311935807426, + "loss": 2.7064, + "theoretical_loss": 3.766933736076864, + "tokens_seen": 723320832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943630892678034, + "loss": 2.7234, + "theoretical_loss": 3.766899285711132, + "tokens_seen": 723386368 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003943530591775326, + "loss": 3.1493, + "theoretical_loss": 3.766864839340138, + "tokens_seen": 723451904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039434302908726175, + "loss": 2.6272, + "theoretical_loss": 3.7668303969630554, + "tokens_seen": 723517440 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394332998996991, + "loss": 2.9986, + "theoretical_loss": 3.7667959585790607, + "tokens_seen": 723582976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039432296890672017, + "loss": 2.8482, + "theoretical_loss": 3.76676152418733, + "tokens_seen": 723648512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039431293881644935, + "loss": 2.808, + "theoretical_loss": 3.7667270937870363, + "tokens_seen": 723714048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039430290872617853, + "loss": 2.7588, + "theoretical_loss": 3.7666926673773586, + "tokens_seen": 723779584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039429287863590776, + "loss": 2.6425, + "theoretical_loss": 3.7666582449574717, + "tokens_seen": 723845120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942828485456369, + "loss": 2.6043, + "theoretical_loss": 3.766623826526552, + "tokens_seen": 723910656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942728184553661, + "loss": 2.3893, + "theoretical_loss": 3.766589412083776, + "tokens_seen": 723976192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039426278836509525, + "loss": 2.8363, + "theoretical_loss": 3.766555001628322, + "tokens_seen": 724041728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942527582748245, + "loss": 2.7142, + "theoretical_loss": 3.766520595159365, + "tokens_seen": 724107264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 408057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.163313150405884, + "objective/train/theoretical_loss": 3.766486192676084, + "objective/train/tokens_used": 744632800, + "theoretical_loss": 3.766486192676084, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039424272818455367, + "loss": 2.7706, + "theoretical_loss": 3.766486192676084, + "tokens_seen": 724172800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039423269809428285, + "loss": 2.7204, + "theoretical_loss": 3.766451794177656, + "tokens_seen": 724238336 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039422266800401203, + "loss": 2.8968, + "theoretical_loss": 3.7664173996632595, + "tokens_seen": 724303872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942126379137412, + "loss": 2.7789, + "theoretical_loss": 3.7663830091320722, + "tokens_seen": 724369408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003942026078234704, + "loss": 2.8649, + "theoretical_loss": 3.7663486225832727, + "tokens_seen": 724434944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039419257773319963, + "loss": 3.0287, + "theoretical_loss": 3.7663142400160403, + "tokens_seen": 724500480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039418254764292876, + "loss": 2.9706, + "theoretical_loss": 3.7662798614295516, + "tokens_seen": 724566016 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394172517552658, + "loss": 2.5185, + "theoretical_loss": 3.7662454868229887, + "tokens_seen": 724631552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941624874623871, + "loss": 2.6261, + "theoretical_loss": 3.766211116195528, + "tokens_seen": 724697088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039415245737211635, + "loss": 2.7275, + "theoretical_loss": 3.766176749546352, + "tokens_seen": 724762624 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941424272818456, + "loss": 2.8693, + "theoretical_loss": 3.7661423868746384, + "tokens_seen": 724828160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941323971915747, + "loss": 2.6889, + "theoretical_loss": 3.766108028179568, + "tokens_seen": 724893696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039412236710130395, + "loss": 2.7965, + "theoretical_loss": 3.7660736734603217, + "tokens_seen": 724959232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039411233701103313, + "loss": 2.7244, + "theoretical_loss": 3.7660393227160798, + "tokens_seen": 725024768 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003941023069207623, + "loss": 2.8347, + "theoretical_loss": 3.7660049759460223, + "tokens_seen": 725090304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940922768304915, + "loss": 2.7925, + "theoretical_loss": 3.7659706331493314, + "tokens_seen": 725155840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940822467402207, + "loss": 2.8973, + "theoretical_loss": 3.7659362943251877, + "tokens_seen": 725221376 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039407221664994986, + "loss": 2.4894, + "theoretical_loss": 3.765901959472773, + "tokens_seen": 725286912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940621865596791, + "loss": 2.7852, + "theoretical_loss": 3.7658676285912693, + "tokens_seen": 725352448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940521564694082, + "loss": 2.8938, + "theoretical_loss": 3.765833301679858, + "tokens_seen": 725417984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039404212637913745, + "loss": 2.7822, + "theoretical_loss": 3.765798978737722, + "tokens_seen": 725483520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940320962888666, + "loss": 2.7152, + "theoretical_loss": 3.765764659764044, + "tokens_seen": 725549056 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940220661985958, + "loss": 2.7457, + "theoretical_loss": 3.7657303447580066, + "tokens_seen": 725614592 + }, + { + "epoch": 0.22, + "learning_rate": 0.000394012036108325, + "loss": 2.7628, + "theoretical_loss": 3.765696033718792, + "tokens_seen": 725680128 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003940020060180542, + "loss": 2.8026, + "theoretical_loss": 3.7656617266455843, + "tokens_seen": 725745664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 409133, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.649945020675659, + "objective/train/theoretical_loss": 3.765627423537567, + "objective/train/tokens_used": 746271200, + "theoretical_loss": 3.765627423537567, + "tokens_seen": 725811200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039399197592778336, + "loss": 2.6373, + "theoretical_loss": 3.765627423537567, + "tokens_seen": 725811200 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939819458375126, + "loss": 2.9122, + "theoretical_loss": 3.765593124393923, + "tokens_seen": 725876736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939719157472417, + "loss": 2.8708, + "theoretical_loss": 3.7655588292138376, + "tokens_seen": 725942272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039396188565697096, + "loss": 2.7742, + "theoretical_loss": 3.7655245379964937, + "tokens_seen": 726007808 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939518555667001, + "loss": 2.6562, + "theoretical_loss": 3.765490250741077, + "tokens_seen": 726073344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939418254764293, + "loss": 2.6231, + "theoretical_loss": 3.765455967446771, + "tokens_seen": 726138880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939317953861585, + "loss": 2.758, + "theoretical_loss": 3.7654216881127613, + "tokens_seen": 726204416 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003939217652958877, + "loss": 2.7845, + "theoretical_loss": 3.7653874127382334, + "tokens_seen": 726269952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039391173520561686, + "loss": 2.8462, + "theoretical_loss": 3.765353141322372, + "tokens_seen": 726335488 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039390170511534604, + "loss": 2.6643, + "theoretical_loss": 3.7653188738643637, + "tokens_seen": 726401024 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938916750250752, + "loss": 2.9446, + "theoretical_loss": 3.765284610363393, + "tokens_seen": 726466560 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039388164493480446, + "loss": 2.9237, + "theoretical_loss": 3.765250350818647, + "tokens_seen": 726532096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938716148445336, + "loss": 2.5889, + "theoretical_loss": 3.7652160952293126, + "tokens_seen": 726597632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938615847542628, + "loss": 2.7943, + "theoretical_loss": 3.765181843594575, + "tokens_seen": 726663168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039385155466399195, + "loss": 2.8595, + "theoretical_loss": 3.7651475959136222, + "tokens_seen": 726728704 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003938415245737212, + "loss": 2.7223, + "theoretical_loss": 3.765113352185641, + "tokens_seen": 726794240 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039383149448345037, + "loss": 2.4638, + "theoretical_loss": 3.7650791124098193, + "tokens_seen": 726859776 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039382146439317955, + "loss": 2.8602, + "theoretical_loss": 3.7650448765853435, + "tokens_seen": 726925312 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039381143430290873, + "loss": 2.732, + "theoretical_loss": 3.765010644711402, + "tokens_seen": 726990848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039380140421263796, + "loss": 2.8741, + "theoretical_loss": 3.7649764167871824, + "tokens_seen": 727056384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937913741223671, + "loss": 2.7782, + "theoretical_loss": 3.7649421928118745, + "tokens_seen": 727121920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937813440320963, + "loss": 2.6986, + "theoretical_loss": 3.7649079727846653, + "tokens_seen": 727187456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039377131394182545, + "loss": 3.0349, + "theoretical_loss": 3.764873756704745, + "tokens_seen": 727252992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937612838515547, + "loss": 2.8043, + "theoretical_loss": 3.764839544571301, + "tokens_seen": 727318528 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039375125376128387, + "loss": 2.7541, + "theoretical_loss": 3.7648053363835237, + "tokens_seen": 727384064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 409693, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.393568992614746, + "objective/train/theoretical_loss": 3.764771132140602, + "objective/train/tokens_used": 747909600, + "theoretical_loss": 3.764771132140602, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039374122367101305, + "loss": 2.8103, + "theoretical_loss": 3.764771132140602, + "tokens_seen": 727449600 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039373119358074223, + "loss": 2.768, + "theoretical_loss": 3.7647369318417265, + "tokens_seen": 727515136 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937211634904714, + "loss": 2.853, + "theoretical_loss": 3.764702735486086, + "tokens_seen": 727580672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003937111334002006, + "loss": 2.7399, + "theoretical_loss": 3.764668543072872, + "tokens_seen": 727646208 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039370110330992983, + "loss": 2.8607, + "theoretical_loss": 3.764634354601274, + "tokens_seen": 727711744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039369107321965896, + "loss": 2.9141, + "theoretical_loss": 3.7646001700704836, + "tokens_seen": 727777280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936810431293882, + "loss": 2.9176, + "theoretical_loss": 3.7645659894796903, + "tokens_seen": 727842816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936710130391173, + "loss": 2.7348, + "theoretical_loss": 3.7645318128280865, + "tokens_seen": 727908352 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039366098294884655, + "loss": 2.9764, + "theoretical_loss": 3.7644976401148638, + "tokens_seen": 727973888 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039365095285857573, + "loss": 2.6341, + "theoretical_loss": 3.764463471339213, + "tokens_seen": 728039424 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936409227683049, + "loss": 2.823, + "theoretical_loss": 3.7644293065003263, + "tokens_seen": 728104960 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936308926780341, + "loss": 2.6593, + "theoretical_loss": 3.764395145597396, + "tokens_seen": 728170496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039362086258776333, + "loss": 2.7037, + "theoretical_loss": 3.7643609886296145, + "tokens_seen": 728236032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039361083249749246, + "loss": 2.6955, + "theoretical_loss": 3.764326835596174, + "tokens_seen": 728301568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003936008024072217, + "loss": 2.6841, + "theoretical_loss": 3.7642926864962685, + "tokens_seen": 728367104 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935907723169508, + "loss": 2.8496, + "theoretical_loss": 3.76425854132909, + "tokens_seen": 728432640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039358074222668006, + "loss": 2.7119, + "theoretical_loss": 3.7642244000938314, + "tokens_seen": 728498176 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039357071213640924, + "loss": 2.7208, + "theoretical_loss": 3.764190262789688, + "tokens_seen": 728563712 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935606820461384, + "loss": 2.5211, + "theoretical_loss": 3.764156129415852, + "tokens_seen": 728629248 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935506519558676, + "loss": 2.6906, + "theoretical_loss": 3.7641219999715174, + "tokens_seen": 728694784 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935406218655968, + "loss": 2.8027, + "theoretical_loss": 3.76408787445588, + "tokens_seen": 728760320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039353059177532596, + "loss": 2.7496, + "theoretical_loss": 3.7640537528681333, + "tokens_seen": 728825856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935205616850552, + "loss": 2.7888, + "theoretical_loss": 3.7640196352074726, + "tokens_seen": 728891392 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003935105315947843, + "loss": 2.8002, + "theoretical_loss": 3.7639855214730917, + "tokens_seen": 728956928 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039350050150451356, + "loss": 2.8055, + "theoretical_loss": 3.763951411664187, + "tokens_seen": 729022464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 410670, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.842893123626709, + "objective/train/theoretical_loss": 3.7639173057799535, + "objective/train/tokens_used": 749548000, + "theoretical_loss": 3.7639173057799535, + "tokens_seen": 729088000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934904714142427, + "loss": 2.8283, + "theoretical_loss": 3.7639173057799535, + "tokens_seen": 729088000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934804413239719, + "loss": 2.8309, + "theoretical_loss": 3.7638832038195877, + "tokens_seen": 729153536 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934704112337011, + "loss": 2.8713, + "theoretical_loss": 3.763849105782284, + "tokens_seen": 729219072 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934603811434303, + "loss": 2.6372, + "theoretical_loss": 3.7638150116672398, + "tokens_seen": 729284608 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039345035105315946, + "loss": 2.6612, + "theoretical_loss": 3.763780921473651, + "tokens_seen": 729350144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934403209628887, + "loss": 2.7433, + "theoretical_loss": 3.7637468352007146, + "tokens_seen": 729415680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039343029087261783, + "loss": 2.681, + "theoretical_loss": 3.7637127528476277, + "tokens_seen": 729481216 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039342026078234706, + "loss": 2.8188, + "theoretical_loss": 3.763678674413586, + "tokens_seen": 729546752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934102306920762, + "loss": 2.8761, + "theoretical_loss": 3.7636445998977885, + "tokens_seen": 729612288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003934002006018054, + "loss": 2.7117, + "theoretical_loss": 3.7636105292994326, + "tokens_seen": 729677824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039339017051153466, + "loss": 3.0125, + "theoretical_loss": 3.7635764626177153, + "tokens_seen": 729743360 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933801404212638, + "loss": 2.9, + "theoretical_loss": 3.7635423998518354, + "tokens_seen": 729808896 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393370110330993, + "loss": 3.011, + "theoretical_loss": 3.763508341000991, + "tokens_seen": 729874432 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039336008024072215, + "loss": 2.765, + "theoretical_loss": 3.76347428606438, + "tokens_seen": 729939968 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003933500501504514, + "loss": 2.8786, + "theoretical_loss": 3.7634402350412026, + "tokens_seen": 730005504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039334002006018057, + "loss": 2.7195, + "theoretical_loss": 3.7634061879306566, + "tokens_seen": 730071040 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039332998996990975, + "loss": 2.8264, + "theoretical_loss": 3.7633721447319415, + "tokens_seen": 730136576 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039331995987963893, + "loss": 2.8929, + "theoretical_loss": 3.763338105444257, + "tokens_seen": 730202112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039330992978936816, + "loss": 2.9521, + "theoretical_loss": 3.763304070066803, + "tokens_seen": 730267648 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932998996990973, + "loss": 2.9408, + "theoretical_loss": 3.7632700385987787, + "tokens_seen": 730333184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932898696088265, + "loss": 2.7265, + "theoretical_loss": 3.7632360110393854, + "tokens_seen": 730398720 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039327983951855565, + "loss": 2.6464, + "theoretical_loss": 3.7632019873878226, + "tokens_seen": 730464256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932698094282849, + "loss": 2.8734, + "theoretical_loss": 3.763167967643292, + "tokens_seen": 730529792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039325977933801407, + "loss": 2.8426, + "theoretical_loss": 3.763133951804993, + "tokens_seen": 730595328 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039324974924774325, + "loss": 2.6467, + "theoretical_loss": 3.763099939872128, + "tokens_seen": 730660864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 411160, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.812192678451538, + "objective/train/theoretical_loss": 3.763065931843898, + "objective/train/tokens_used": 751186400, + "theoretical_loss": 3.763065931843898, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039323971915747243, + "loss": 2.8428, + "theoretical_loss": 3.763065931843898, + "tokens_seen": 730726400 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932296890672016, + "loss": 2.6838, + "theoretical_loss": 3.7630319277195046, + "tokens_seen": 730791936 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003932196589769308, + "loss": 3.172, + "theoretical_loss": 3.7629979274981498, + "tokens_seen": 730857472 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039320962888666003, + "loss": 2.6505, + "theoretical_loss": 3.7629639311790353, + "tokens_seen": 730923008 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039319959879638916, + "loss": 2.8953, + "theoretical_loss": 3.762929938761363, + "tokens_seen": 730988544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931895687061184, + "loss": 2.8217, + "theoretical_loss": 3.762895950244337, + "tokens_seen": 731054080 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931795386158475, + "loss": 2.6984, + "theoretical_loss": 3.7628619656271587, + "tokens_seen": 731119616 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039316950852557675, + "loss": 2.958, + "theoretical_loss": 3.7628279849090314, + "tokens_seen": 731185152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039315947843530593, + "loss": 2.8194, + "theoretical_loss": 3.762794008089158, + "tokens_seen": 731250688 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931494483450351, + "loss": 2.8385, + "theoretical_loss": 3.7627600351667434, + "tokens_seen": 731316224 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931394182547643, + "loss": 2.7625, + "theoretical_loss": 3.7627260661409903, + "tokens_seen": 731381760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039312938816449353, + "loss": 2.675, + "theoretical_loss": 3.762692101011103, + "tokens_seen": 731447296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039311935807422266, + "loss": 2.8194, + "theoretical_loss": 3.762658139776285, + "tokens_seen": 731512832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003931093279839519, + "loss": 2.9804, + "theoretical_loss": 3.762624182435741, + "tokens_seen": 731578368 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393099297893681, + "loss": 2.8233, + "theoretical_loss": 3.7625902289886763, + "tokens_seen": 731643904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039308926780341026, + "loss": 3.0054, + "theoretical_loss": 3.762556279434295, + "tokens_seen": 731709440 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039307923771313944, + "loss": 2.9656, + "theoretical_loss": 3.7625223337718023, + "tokens_seen": 731774976 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930692076228686, + "loss": 2.7612, + "theoretical_loss": 3.762488392000404, + "tokens_seen": 731840512 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930591775325978, + "loss": 2.9062, + "theoretical_loss": 3.7624544541193057, + "tokens_seen": 731906048 + }, + { + "epoch": 0.22, + "learning_rate": 0.000393049147442327, + "loss": 2.5917, + "theoretical_loss": 3.762420520127713, + "tokens_seen": 731971584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039303911735205616, + "loss": 2.8431, + "theoretical_loss": 3.762386590024832, + "tokens_seen": 732037120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930290872617854, + "loss": 2.8048, + "theoretical_loss": 3.7623526638098683, + "tokens_seen": 732102656 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003930190571715145, + "loss": 2.6144, + "theoretical_loss": 3.76231874148203, + "tokens_seen": 732168192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039300902708124376, + "loss": 2.6214, + "theoretical_loss": 3.762284823040522, + "tokens_seen": 732233728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929989969909729, + "loss": 2.866, + "theoretical_loss": 3.762250908484553, + "tokens_seen": 732299264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 412414, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.900200366973877, + "objective/train/theoretical_loss": 3.7622169978133293, + "objective/train/tokens_used": 752824800, + "theoretical_loss": 3.7622169978133293, + "tokens_seen": 732364800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929889669007021, + "loss": 3.019, + "theoretical_loss": 3.7622169978133293, + "tokens_seen": 732364800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929789368104313, + "loss": 2.8027, + "theoretical_loss": 3.762183091026058, + "tokens_seen": 732430336 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929689067201605, + "loss": 2.7653, + "theoretical_loss": 3.7621491881219478, + "tokens_seen": 732495872 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039295887662988967, + "loss": 2.5643, + "theoretical_loss": 3.7621152891002057, + "tokens_seen": 732561408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929488465396189, + "loss": 2.5249, + "theoretical_loss": 3.7620813939600404, + "tokens_seen": 732626944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039293881644934803, + "loss": 2.7563, + "theoretical_loss": 3.76204750270066, + "tokens_seen": 732692480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039292878635907726, + "loss": 2.6964, + "theoretical_loss": 3.7620136153212735, + "tokens_seen": 732758016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929187562688064, + "loss": 2.9321, + "theoretical_loss": 3.7619797318210892, + "tokens_seen": 732823552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003929087261785356, + "loss": 2.9943, + "theoretical_loss": 3.7619458521993168, + "tokens_seen": 732889088 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928986960882648, + "loss": 3.0453, + "theoretical_loss": 3.761911976455165, + "tokens_seen": 732954624 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392888665997994, + "loss": 2.7816, + "theoretical_loss": 3.7618781045878436, + "tokens_seen": 733020160 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039287863590772317, + "loss": 2.9523, + "theoretical_loss": 3.7618442365965628, + "tokens_seen": 733085696 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039286860581745235, + "loss": 2.7939, + "theoretical_loss": 3.7618103724805314, + "tokens_seen": 733151232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039285857572718153, + "loss": 2.8391, + "theoretical_loss": 3.7617765122389613, + "tokens_seen": 733216768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039284854563691077, + "loss": 2.8972, + "theoretical_loss": 3.7617426558710614, + "tokens_seen": 733282304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928385155466399, + "loss": 2.7244, + "theoretical_loss": 3.7617088033760435, + "tokens_seen": 733347840 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039282848545636913, + "loss": 2.5487, + "theoretical_loss": 3.7616749547531185, + "tokens_seen": 733413376 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039281845536609825, + "loss": 2.4826, + "theoretical_loss": 3.7616411100014964, + "tokens_seen": 733478912 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003928084252758275, + "loss": 2.7962, + "theoretical_loss": 3.7616072691203897, + "tokens_seen": 733544448 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039279839518555667, + "loss": 2.9535, + "theoretical_loss": 3.76157343210901, + "tokens_seen": 733609984 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039278836509528585, + "loss": 2.9491, + "theoretical_loss": 3.7615395989665688, + "tokens_seen": 733675520 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039277833500501503, + "loss": 2.9444, + "theoretical_loss": 3.7615057696922785, + "tokens_seen": 733741056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039276830491474427, + "loss": 2.8252, + "theoretical_loss": 3.7614719442853506, + "tokens_seen": 733806592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927582748244734, + "loss": 2.7134, + "theoretical_loss": 3.7614381227449982, + "tokens_seen": 733872128 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039274824473420263, + "loss": 2.8441, + "theoretical_loss": 3.7614043050704344, + "tokens_seen": 733937664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 413470, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.670003652572632, + "objective/train/theoretical_loss": 3.7613704912608723, + "objective/train/tokens_used": 754463200, + "theoretical_loss": 3.7613704912608723, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039273821464393176, + "loss": 2.9001, + "theoretical_loss": 3.7613704912608723, + "tokens_seen": 734003200 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392728184553661, + "loss": 2.7337, + "theoretical_loss": 3.761336681315524, + "tokens_seen": 734068736 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003927181544633902, + "loss": 2.5806, + "theoretical_loss": 3.7613028752336044, + "tokens_seen": 734134272 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039270812437311936, + "loss": 2.8466, + "theoretical_loss": 3.761269073014326, + "tokens_seen": 734199808 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039269809428284854, + "loss": 2.7271, + "theoretical_loss": 3.7612352746569035, + "tokens_seen": 734265344 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926880641925777, + "loss": 2.8845, + "theoretical_loss": 3.761201480160551, + "tokens_seen": 734330880 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926780341023069, + "loss": 2.6129, + "theoretical_loss": 3.7611676895244828, + "tokens_seen": 734396416 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039266800401203613, + "loss": 2.646, + "theoretical_loss": 3.761133902747913, + "tokens_seen": 734461952 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039265797392176526, + "loss": 2.6668, + "theoretical_loss": 3.7611001198300578, + "tokens_seen": 734527488 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926479438314945, + "loss": 2.625, + "theoretical_loss": 3.7610663407701304, + "tokens_seen": 734593024 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039263791374122373, + "loss": 2.9871, + "theoretical_loss": 3.7610325655673478, + "tokens_seen": 734658560 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039262788365095286, + "loss": 2.8132, + "theoretical_loss": 3.7609987942209244, + "tokens_seen": 734724096 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926178535606821, + "loss": 2.7216, + "theoretical_loss": 3.7609650267300765, + "tokens_seen": 734789632 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003926078234704112, + "loss": 2.7455, + "theoretical_loss": 3.7609312630940197, + "tokens_seen": 734855168 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039259779338014046, + "loss": 2.6653, + "theoretical_loss": 3.7608975033119707, + "tokens_seen": 734920704 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039258776328986964, + "loss": 2.6471, + "theoretical_loss": 3.760863747383146, + "tokens_seen": 734986240 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925777331995988, + "loss": 2.8097, + "theoretical_loss": 3.7608299953067625, + "tokens_seen": 735051776 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392567703109328, + "loss": 2.8716, + "theoretical_loss": 3.760796247082036, + "tokens_seen": 735117312 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925576730190572, + "loss": 2.8082, + "theoretical_loss": 3.760762502708185, + "tokens_seen": 735182848 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039254764292878636, + "loss": 2.6588, + "theoretical_loss": 3.760728762184425, + "tokens_seen": 735248384 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925376128385156, + "loss": 2.731, + "theoretical_loss": 3.7606950255099765, + "tokens_seen": 735313920 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925275827482447, + "loss": 2.8463, + "theoretical_loss": 3.7606612926840546, + "tokens_seen": 735379456 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039251755265797396, + "loss": 2.6797, + "theoretical_loss": 3.760627563705879, + "tokens_seen": 735444992 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003925075225677031, + "loss": 2.7702, + "theoretical_loss": 3.7605938385746667, + "tokens_seen": 735510528 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924974924774323, + "loss": 2.8931, + "theoretical_loss": 3.7605601172896366, + "tokens_seen": 735576064 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 413970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.987671375274658, + "objective/train/theoretical_loss": 3.7605263998500087, + "objective/train/tokens_used": 756101600, + "theoretical_loss": 3.7605263998500087, + "tokens_seen": 735641600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924874623871615, + "loss": 2.9723, + "theoretical_loss": 3.7605263998500087, + "tokens_seen": 735641600 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924774322968907, + "loss": 2.759, + "theoretical_loss": 3.7604926862550005, + "tokens_seen": 735707136 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039246740220661987, + "loss": 3.0025, + "theoretical_loss": 3.7604589765038323, + "tokens_seen": 735772672 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924573721163491, + "loss": 2.7319, + "theoretical_loss": 3.7604252705957224, + "tokens_seen": 735838208 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039244734202607823, + "loss": 2.8027, + "theoretical_loss": 3.7603915685298914, + "tokens_seen": 735903744 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039243731193580746, + "loss": 2.7283, + "theoretical_loss": 3.7603578703055587, + "tokens_seen": 735969280 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924272818455366, + "loss": 2.699, + "theoretical_loss": 3.7603241759219443, + "tokens_seen": 736034816 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003924172517552658, + "loss": 2.9509, + "theoretical_loss": 3.7602904853782695, + "tokens_seen": 736100352 + }, + { + "epoch": 0.22, + "learning_rate": 0.000392407221664995, + "loss": 2.468, + "theoretical_loss": 3.7602567986737534, + "tokens_seen": 736165888 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923971915747242, + "loss": 2.6287, + "theoretical_loss": 3.760223115807618, + "tokens_seen": 736231424 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039238716148445337, + "loss": 2.8342, + "theoretical_loss": 3.7601894367790836, + "tokens_seen": 736296960 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039237713139418255, + "loss": 2.7653, + "theoretical_loss": 3.760155761587372, + "tokens_seen": 736362496 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039236710130391173, + "loss": 3.1294, + "theoretical_loss": 3.760122090231704, + "tokens_seen": 736428032 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039235707121364097, + "loss": 2.7462, + "theoretical_loss": 3.760088422711302, + "tokens_seen": 736493568 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923470411233701, + "loss": 2.7666, + "theoretical_loss": 3.7600547590253877, + "tokens_seen": 736559104 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039233701103309933, + "loss": 2.5778, + "theoretical_loss": 3.7600210991731826, + "tokens_seen": 736624640 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039232698094282846, + "loss": 2.6942, + "theoretical_loss": 3.75998744315391, + "tokens_seen": 736690176 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003923169508525577, + "loss": 2.8405, + "theoretical_loss": 3.759953790966793, + "tokens_seen": 736755712 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039230692076228687, + "loss": 2.7331, + "theoretical_loss": 3.759920142611053, + "tokens_seen": 736821248 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039229689067201605, + "loss": 2.8218, + "theoretical_loss": 3.7598864980859132, + "tokens_seen": 736886784 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039228686058174523, + "loss": 2.6316, + "theoretical_loss": 3.759852857390598, + "tokens_seen": 736952320 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039227683049147447, + "loss": 2.8765, + "theoretical_loss": 3.75981922052433, + "tokens_seen": 737017856 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922668004012036, + "loss": 2.7292, + "theoretical_loss": 3.7597855874863333, + "tokens_seen": 737083392 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039225677031093283, + "loss": 2.7402, + "theoretical_loss": 3.7597519582758325, + "tokens_seen": 737148928 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039224674022066196, + "loss": 2.6244, + "theoretical_loss": 3.7597183328920507, + "tokens_seen": 737214464 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 414896, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8144378662109375, + "objective/train/theoretical_loss": 3.759684711334213, + "objective/train/tokens_used": 757740000, + "theoretical_loss": 3.759684711334213, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922367101303912, + "loss": 2.7107, + "theoretical_loss": 3.759684711334213, + "tokens_seen": 737280000 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003922266800401204, + "loss": 2.8554, + "theoretical_loss": 3.7596510936015433, + "tokens_seen": 737345536 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039221664994984956, + "loss": 2.7267, + "theoretical_loss": 3.759617479693267, + "tokens_seen": 737411072 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039220661985957874, + "loss": 2.9452, + "theoretical_loss": 3.75958386960861, + "tokens_seen": 737476608 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921965897693079, + "loss": 2.7452, + "theoretical_loss": 3.759550263346796, + "tokens_seen": 737542144 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921865596790371, + "loss": 3.0845, + "theoretical_loss": 3.759516660907052, + "tokens_seen": 737607680 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039217652958876633, + "loss": 2.5286, + "theoretical_loss": 3.759483062288603, + "tokens_seen": 737673216 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039216649949849546, + "loss": 2.8046, + "theoretical_loss": 3.759449467490675, + "tokens_seen": 737738752 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921564694082247, + "loss": 2.9167, + "theoretical_loss": 3.7594158765124943, + "tokens_seen": 737804288 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921464393179539, + "loss": 2.9723, + "theoretical_loss": 3.759382289353288, + "tokens_seen": 737869824 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039213640922768306, + "loss": 2.7637, + "theoretical_loss": 3.759348706012282, + "tokens_seen": 737935360 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039212637913741224, + "loss": 2.8318, + "theoretical_loss": 3.759315126488704, + "tokens_seen": 738000896 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921163490471414, + "loss": 2.885, + "theoretical_loss": 3.75928155078178, + "tokens_seen": 738066432 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003921063189568706, + "loss": 2.8873, + "theoretical_loss": 3.7592479788907376, + "tokens_seen": 738131968 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039209628886659984, + "loss": 2.8126, + "theoretical_loss": 3.7592144108148053, + "tokens_seen": 738197504 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039208625877632896, + "loss": 2.7385, + "theoretical_loss": 3.7591808465532104, + "tokens_seen": 738263040 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920762286860582, + "loss": 2.6291, + "theoretical_loss": 3.759147286105181, + "tokens_seen": 738328576 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920661985957873, + "loss": 2.9438, + "theoretical_loss": 3.7591137294699446, + "tokens_seen": 738394112 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039205616850551656, + "loss": 2.8742, + "theoretical_loss": 3.759080176646731, + "tokens_seen": 738459648 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039204613841524574, + "loss": 2.5495, + "theoretical_loss": 3.759046627634768, + "tokens_seen": 738525184 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920361083249749, + "loss": 2.5249, + "theoretical_loss": 3.759013082433285, + "tokens_seen": 738590720 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920260782347041, + "loss": 2.7102, + "theoretical_loss": 3.7589795410415103, + "tokens_seen": 738656256 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003920160481444333, + "loss": 2.9134, + "theoretical_loss": 3.758946003458675, + "tokens_seen": 738721792 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039200601805416247, + "loss": 2.8764, + "theoretical_loss": 3.758912469684007, + "tokens_seen": 738787328 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919959879638917, + "loss": 2.7331, + "theoretical_loss": 3.7588789397167366, + "tokens_seen": 738852864 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 415468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.898322820663452, + "objective/train/theoretical_loss": 3.7588454135560943, + "objective/train/tokens_used": 759378400, + "theoretical_loss": 3.7588454135560943, + "tokens_seen": 738918400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039198595787362083, + "loss": 2.7734, + "theoretical_loss": 3.7588454135560943, + "tokens_seen": 738918400 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039197592778335007, + "loss": 2.7841, + "theoretical_loss": 3.7588118912013098, + "tokens_seen": 738983936 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039196589769307925, + "loss": 2.8882, + "theoretical_loss": 3.758778372651614, + "tokens_seen": 739049472 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039195586760280843, + "loss": 2.8642, + "theoretical_loss": 3.7587448579062377, + "tokens_seen": 739115008 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919458375125376, + "loss": 2.7614, + "theoretical_loss": 3.758711346964411, + "tokens_seen": 739180544 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919358074222668, + "loss": 2.8015, + "theoretical_loss": 3.7586778398253666, + "tokens_seen": 739246080 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039192577733199597, + "loss": 2.893, + "theoretical_loss": 3.7586443364883344, + "tokens_seen": 739311616 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003919157472417252, + "loss": 2.9149, + "theoretical_loss": 3.7586108369525473, + "tokens_seen": 739377152 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039190571715145433, + "loss": 2.5778, + "theoretical_loss": 3.7585773412172356, + "tokens_seen": 739442688 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039189568706118357, + "loss": 2.9246, + "theoretical_loss": 3.758543849281633, + "tokens_seen": 739508224 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039188565697091275, + "loss": 2.844, + "theoretical_loss": 3.758510361144971, + "tokens_seen": 739573760 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039187562688064193, + "loss": 2.7873, + "theoretical_loss": 3.7584768768064816, + "tokens_seen": 739639296 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039186559679037117, + "loss": 2.9233, + "theoretical_loss": 3.758443396265399, + "tokens_seen": 739704832 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918555667001003, + "loss": 2.7354, + "theoretical_loss": 3.758409919520955, + "tokens_seen": 739770368 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039184553660982953, + "loss": 2.8651, + "theoretical_loss": 3.7583764465723823, + "tokens_seen": 739835904 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039183550651955866, + "loss": 2.791, + "theoretical_loss": 3.758342977418916, + "tokens_seen": 739901440 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003918254764292879, + "loss": 2.7292, + "theoretical_loss": 3.758309512059788, + "tokens_seen": 739966976 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039181544633901707, + "loss": 2.9291, + "theoretical_loss": 3.758276050494233, + "tokens_seen": 740032512 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039180541624874625, + "loss": 2.8555, + "theoretical_loss": 3.7582425927214853, + "tokens_seen": 740098048 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039179538615847543, + "loss": 2.8255, + "theoretical_loss": 3.7582091387407788, + "tokens_seen": 740163584 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039178535606820467, + "loss": 2.6274, + "theoretical_loss": 3.758175688551348, + "tokens_seen": 740229120 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917753259779338, + "loss": 2.7021, + "theoretical_loss": 3.758142242152428, + "tokens_seen": 740294656 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039176529588766303, + "loss": 2.7977, + "theoretical_loss": 3.758108799543254, + "tokens_seen": 740360192 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039175526579739216, + "loss": 3.0315, + "theoretical_loss": 3.7580753607230593, + "tokens_seen": 740425728 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917452357071214, + "loss": 2.8211, + "theoretical_loss": 3.7580419256910815, + "tokens_seen": 740491264 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 416744, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.713137626647949, + "objective/train/theoretical_loss": 3.7580084944465555, + "objective/train/tokens_used": 761016800, + "theoretical_loss": 3.7580084944465555, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917352056168506, + "loss": 2.8892, + "theoretical_loss": 3.7580084944465555, + "tokens_seen": 740556800 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039172517552657976, + "loss": 2.933, + "theoretical_loss": 3.757975066988717, + "tokens_seen": 740622336 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039171514543630894, + "loss": 2.7738, + "theoretical_loss": 3.7579416433168022, + "tokens_seen": 740687872 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003917051153460381, + "loss": 3.0515, + "theoretical_loss": 3.757908223430047, + "tokens_seen": 740753408 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916950852557673, + "loss": 2.7393, + "theoretical_loss": 3.7578748073276884, + "tokens_seen": 740818944 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039168505516549653, + "loss": 2.5972, + "theoretical_loss": 3.7578413950089633, + "tokens_seen": 740884480 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039167502507522566, + "loss": 2.8358, + "theoretical_loss": 3.757807986473108, + "tokens_seen": 740950016 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916649949849549, + "loss": 2.6884, + "theoretical_loss": 3.75777458171936, + "tokens_seen": 741015552 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916549648946841, + "loss": 2.9793, + "theoretical_loss": 3.757741180746957, + "tokens_seen": 741081088 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039164493480441326, + "loss": 2.938, + "theoretical_loss": 3.7577077835551362, + "tokens_seen": 741146624 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039163490471414244, + "loss": 2.772, + "theoretical_loss": 3.7576743901431353, + "tokens_seen": 741212160 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916248746238716, + "loss": 2.7569, + "theoretical_loss": 3.7576410005101932, + "tokens_seen": 741277696 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003916148445336008, + "loss": 2.74, + "theoretical_loss": 3.757607614655547, + "tokens_seen": 741343232 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039160481444333004, + "loss": 2.8583, + "theoretical_loss": 3.7575742325784365, + "tokens_seen": 741408768 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039159478435305916, + "loss": 2.9713, + "theoretical_loss": 3.7575408542780995, + "tokens_seen": 741474304 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915847542627884, + "loss": 2.6112, + "theoretical_loss": 3.757507479753775, + "tokens_seen": 741539840 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915747241725175, + "loss": 2.6381, + "theoretical_loss": 3.7574741090047024, + "tokens_seen": 741605376 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039156469408224676, + "loss": 2.6271, + "theoretical_loss": 3.7574407420301217, + "tokens_seen": 741670912 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039155466399197594, + "loss": 2.7429, + "theoretical_loss": 3.7574073788292712, + "tokens_seen": 741736448 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915446339017051, + "loss": 2.7142, + "theoretical_loss": 3.7573740194013916, + "tokens_seen": 741801984 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915346038114343, + "loss": 2.7973, + "theoretical_loss": 3.7573406637457225, + "tokens_seen": 741867520 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915245737211635, + "loss": 2.6915, + "theoretical_loss": 3.757307311861505, + "tokens_seen": 741933056 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039151454363089267, + "loss": 2.9268, + "theoretical_loss": 3.757273963747979, + "tokens_seen": 741998592 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003915045135406219, + "loss": 2.5838, + "theoretical_loss": 3.7572406194043846, + "tokens_seen": 742064128 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039149448345035103, + "loss": 2.7691, + "theoretical_loss": 3.7572072788299637, + "tokens_seen": 742129664 + }, + { + "epoch": 0.22, + "objective/train/docs_used": 417361, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1539878845214844, + "objective/train/theoretical_loss": 3.7571739420239574, + "objective/train/tokens_used": 762655200, + "theoretical_loss": 3.7571739420239574, + "tokens_seen": 742195200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039148445336008027, + "loss": 2.8494, + "theoretical_loss": 3.7571739420239574, + "tokens_seen": 742195200 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039147442326980945, + "loss": 2.7746, + "theoretical_loss": 3.757140608985606, + "tokens_seen": 742260736 + }, + { + "epoch": 0.22, + "learning_rate": 0.00039146439317953863, + "loss": 2.8907, + "theoretical_loss": 3.7571072797141523, + "tokens_seen": 742326272 + }, + { + "epoch": 0.22, + "learning_rate": 0.0003914543630892678, + "loss": 2.8257, + "theoretical_loss": 3.7570739542088374, + "tokens_seen": 742391808 + }, + { + "epoch": 0.22, + "learning_rate": 0.000391444332998997, + "loss": 2.9461, + "theoretical_loss": 3.757040632468904, + "tokens_seen": 742457344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039143430290872617, + "loss": 2.7556, + "theoretical_loss": 3.7570073144935936, + "tokens_seen": 742522880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003914242728184554, + "loss": 2.8871, + "theoretical_loss": 3.7569740002821495, + "tokens_seen": 742588416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039141424272818453, + "loss": 2.8353, + "theoretical_loss": 3.7569406898338134, + "tokens_seen": 742653952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039140421263791377, + "loss": 2.7706, + "theoretical_loss": 3.756907383147829, + "tokens_seen": 742719488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913941825476429, + "loss": 2.7584, + "theoretical_loss": 3.756874080223439, + "tokens_seen": 742785024 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039138415245737213, + "loss": 2.802, + "theoretical_loss": 3.7568407810598865, + "tokens_seen": 742850560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913741223671013, + "loss": 2.6823, + "theoretical_loss": 3.756807485656416, + "tokens_seen": 742916096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913640922768305, + "loss": 2.8226, + "theoretical_loss": 3.7567741940122703, + "tokens_seen": 742981632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913540621865597, + "loss": 2.7535, + "theoretical_loss": 3.7567409061266943, + "tokens_seen": 743047168 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039134403209628886, + "loss": 2.5167, + "theoretical_loss": 3.756707621998931, + "tokens_seen": 743112704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039133400200601804, + "loss": 2.7885, + "theoretical_loss": 3.7566743416282264, + "tokens_seen": 743178240 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039132397191574727, + "loss": 2.6626, + "theoretical_loss": 3.7566410650138238, + "tokens_seen": 743243776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003913139418254764, + "loss": 2.6599, + "theoretical_loss": 3.756607792154969, + "tokens_seen": 743309312 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039130391173520563, + "loss": 2.755, + "theoretical_loss": 3.7565745230509062, + "tokens_seen": 743374848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912938816449348, + "loss": 2.7896, + "theoretical_loss": 3.7565412577008814, + "tokens_seen": 743440384 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391283851554664, + "loss": 2.5392, + "theoretical_loss": 3.7565079961041397, + "tokens_seen": 743505920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912738214643932, + "loss": 2.7874, + "theoretical_loss": 3.7564747382599273, + "tokens_seen": 743571456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039126379137412236, + "loss": 2.5402, + "theoretical_loss": 3.75644148416749, + "tokens_seen": 743636992 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039125376128385154, + "loss": 2.6678, + "theoretical_loss": 3.7564082338260736, + "tokens_seen": 743702528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912437311935808, + "loss": 2.7886, + "theoretical_loss": 3.756374987234925, + "tokens_seen": 743768064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 418521, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7649760246276855, + "objective/train/theoretical_loss": 3.7563417443932905, + "objective/train/tokens_used": 764293600, + "theoretical_loss": 3.7563417443932905, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912337011033099, + "loss": 2.9361, + "theoretical_loss": 3.7563417443932905, + "tokens_seen": 743833600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039122367101303914, + "loss": 2.9609, + "theoretical_loss": 3.7563085053004173, + "tokens_seen": 743899136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039121364092276826, + "loss": 2.7547, + "theoretical_loss": 3.7562752699555517, + "tokens_seen": 743964672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003912036108324975, + "loss": 2.5979, + "theoretical_loss": 3.756242038357942, + "tokens_seen": 744030208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911935807422267, + "loss": 2.9392, + "theoretical_loss": 3.756208810506835, + "tokens_seen": 744095744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039118355065195586, + "loss": 2.6047, + "theoretical_loss": 3.7561755864014787, + "tokens_seen": 744161280 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039117352056168504, + "loss": 2.8347, + "theoretical_loss": 3.756142366041121, + "tokens_seen": 744226816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911634904714143, + "loss": 2.7851, + "theoretical_loss": 3.7561091494250096, + "tokens_seen": 744292352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911534603811434, + "loss": 2.7388, + "theoretical_loss": 3.756075936552394, + "tokens_seen": 744357888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039114343029087264, + "loss": 2.7189, + "theoretical_loss": 3.756042727422521, + "tokens_seen": 744423424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003911334002006018, + "loss": 2.6181, + "theoretical_loss": 3.756009522034641, + "tokens_seen": 744488960 + }, + { + "epoch": 0.23, + "learning_rate": 0.000391123370110331, + "loss": 2.7899, + "theoretical_loss": 3.755976320388002, + "tokens_seen": 744554496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039111334002006024, + "loss": 2.7496, + "theoretical_loss": 3.755943122481854, + "tokens_seen": 744620032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039110330992978936, + "loss": 2.9665, + "theoretical_loss": 3.755909928315446, + "tokens_seen": 744685568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910932798395186, + "loss": 2.9362, + "theoretical_loss": 3.7558767378880273, + "tokens_seen": 744751104 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039108324974924773, + "loss": 2.5904, + "theoretical_loss": 3.755843551198849, + "tokens_seen": 744816640 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039107321965897696, + "loss": 2.6235, + "theoretical_loss": 3.7558103682471593, + "tokens_seen": 744882176 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039106318956870614, + "loss": 2.6325, + "theoretical_loss": 3.7557771890322105, + "tokens_seen": 744947712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910531594784353, + "loss": 2.7591, + "theoretical_loss": 3.755744013553252, + "tokens_seen": 745013248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910431293881645, + "loss": 2.6799, + "theoretical_loss": 3.755710841809534, + "tokens_seen": 745078784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910330992978937, + "loss": 2.7825, + "theoretical_loss": 3.755677673800309, + "tokens_seen": 745144320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039102306920762287, + "loss": 2.7031, + "theoretical_loss": 3.755644509524827, + "tokens_seen": 745209856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003910130391173521, + "loss": 2.6609, + "theoretical_loss": 3.75561134898234, + "tokens_seen": 745275392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039100300902708123, + "loss": 2.5443, + "theoretical_loss": 3.7555781921720994, + "tokens_seen": 745340928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039099297893681047, + "loss": 2.7118, + "theoretical_loss": 3.7555450390933567, + "tokens_seen": 745406464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 419167, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.160205125808716, + "objective/train/theoretical_loss": 3.7555118897453648, + "objective/train/tokens_used": 765932000, + "theoretical_loss": 3.7555118897453648, + "tokens_seen": 745472000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039098294884653965, + "loss": 2.8914, + "theoretical_loss": 3.7555118897453648, + "tokens_seen": 745472000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039097291875626883, + "loss": 2.6525, + "theoretical_loss": 3.755478744127375, + "tokens_seen": 745537536 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390962888665998, + "loss": 2.771, + "theoretical_loss": 3.75544560223864, + "tokens_seen": 745603072 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909528585757272, + "loss": 2.7758, + "theoretical_loss": 3.755412464078413, + "tokens_seen": 745668608 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039094282848545637, + "loss": 2.7658, + "theoretical_loss": 3.755379329645946, + "tokens_seen": 745734144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909327983951856, + "loss": 2.8785, + "theoretical_loss": 3.7553461989404933, + "tokens_seen": 745799680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039092276830491473, + "loss": 2.7476, + "theoretical_loss": 3.7553130719613073, + "tokens_seen": 745865216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039091273821464397, + "loss": 2.5613, + "theoretical_loss": 3.755279948707642, + "tokens_seen": 745930752 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003909027081243731, + "loss": 2.7278, + "theoretical_loss": 3.7552468291787506, + "tokens_seen": 745996288 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039089267803410233, + "loss": 2.4587, + "theoretical_loss": 3.7552137133738874, + "tokens_seen": 746061824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908826479438315, + "loss": 2.873, + "theoretical_loss": 3.755180601292307, + "tokens_seen": 746127360 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908726178535607, + "loss": 2.8608, + "theoretical_loss": 3.7551474929332636, + "tokens_seen": 746192896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908625877632899, + "loss": 2.9343, + "theoretical_loss": 3.755114388296011, + "tokens_seen": 746258432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039085255767301906, + "loss": 2.5345, + "theoretical_loss": 3.7550812873798054, + "tokens_seen": 746323968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039084252758274824, + "loss": 2.5794, + "theoretical_loss": 3.755048190183901, + "tokens_seen": 746389504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039083249749247747, + "loss": 2.5169, + "theoretical_loss": 3.755015096707553, + "tokens_seen": 746455040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003908224674022066, + "loss": 2.7485, + "theoretical_loss": 3.754982006950017, + "tokens_seen": 746520576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039081243731193583, + "loss": 2.6492, + "theoretical_loss": 3.7549489209105493, + "tokens_seen": 746586112 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390802407221665, + "loss": 2.4441, + "theoretical_loss": 3.754915838588405, + "tokens_seen": 746651648 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907923771313942, + "loss": 2.6574, + "theoretical_loss": 3.7548827599828405, + "tokens_seen": 746717184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907823470411234, + "loss": 2.8731, + "theoretical_loss": 3.754849685093112, + "tokens_seen": 746782720 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039077231695085256, + "loss": 2.6081, + "theoretical_loss": 3.754816613918476, + "tokens_seen": 746848256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039076228686058174, + "loss": 2.744, + "theoretical_loss": 3.7547835464581896, + "tokens_seen": 746913792 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390752256770311, + "loss": 2.9731, + "theoretical_loss": 3.7547504827115104, + "tokens_seen": 746979328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907422266800401, + "loss": 2.9428, + "theoretical_loss": 3.754717422677694, + "tokens_seen": 747044864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 420047, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9569199085235596, + "objective/train/theoretical_loss": 3.754684366355999, + "objective/train/tokens_used": 767570400, + "theoretical_loss": 3.754684366355999, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039073219658976934, + "loss": 2.6868, + "theoretical_loss": 3.754684366355999, + "tokens_seen": 747110400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039072216649949846, + "loss": 2.8481, + "theoretical_loss": 3.7546513137456823, + "tokens_seen": 747175936 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907121364092277, + "loss": 2.6763, + "theoretical_loss": 3.7546182648460023, + "tokens_seen": 747241472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003907021063189569, + "loss": 2.541, + "theoretical_loss": 3.754585219656217, + "tokens_seen": 747307008 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039069207622868606, + "loss": 2.811, + "theoretical_loss": 3.754552178175584, + "tokens_seen": 747372544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039068204613841524, + "loss": 2.8739, + "theoretical_loss": 3.7545191404033624, + "tokens_seen": 747438080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906720160481445, + "loss": 2.7239, + "theoretical_loss": 3.7544861063388106, + "tokens_seen": 747503616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906619859578736, + "loss": 2.7788, + "theoretical_loss": 3.7544530759811883, + "tokens_seen": 747569152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039065195586760284, + "loss": 2.7612, + "theoretical_loss": 3.7544200493297533, + "tokens_seen": 747634688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039064192577733197, + "loss": 2.4776, + "theoretical_loss": 3.7543870263837658, + "tokens_seen": 747700224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906318956870612, + "loss": 2.6039, + "theoretical_loss": 3.754354007142485, + "tokens_seen": 747765760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003906218655967904, + "loss": 2.8779, + "theoretical_loss": 3.754320991605171, + "tokens_seen": 747831296 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039061183550651957, + "loss": 2.6714, + "theoretical_loss": 3.754287979771083, + "tokens_seen": 747896832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039060180541624875, + "loss": 2.8675, + "theoretical_loss": 3.754254971639482, + "tokens_seen": 747962368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039059177532597793, + "loss": 2.8031, + "theoretical_loss": 3.754221967209628, + "tokens_seen": 748027904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905817452357071, + "loss": 2.5851, + "theoretical_loss": 3.754188966480782, + "tokens_seen": 748093440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039057171514543634, + "loss": 2.6848, + "theoretical_loss": 3.754155969452204, + "tokens_seen": 748158976 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039056168505516547, + "loss": 2.8457, + "theoretical_loss": 3.7541229761231563, + "tokens_seen": 748224512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905516549648947, + "loss": 2.8551, + "theoretical_loss": 3.7540899864928985, + "tokens_seen": 748290048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039054162487462383, + "loss": 2.9909, + "theoretical_loss": 3.7540570005606932, + "tokens_seen": 748355584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039053159478435307, + "loss": 2.5867, + "theoretical_loss": 3.754024018325802, + "tokens_seen": 748421120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039052156469408225, + "loss": 2.7651, + "theoretical_loss": 3.7539910397874863, + "tokens_seen": 748486656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039051153460381143, + "loss": 2.7933, + "theoretical_loss": 3.7539580649450084, + "tokens_seen": 748552192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003905015045135406, + "loss": 2.9215, + "theoretical_loss": 3.7539250937976307, + "tokens_seen": 748617728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039049147442326985, + "loss": 2.6618, + "theoretical_loss": 3.7538921263446157, + "tokens_seen": 748683264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 421176, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7754266262054443, + "objective/train/theoretical_loss": 3.753859162585226, + "objective/train/tokens_used": 769208800, + "theoretical_loss": 3.753859162585226, + "tokens_seen": 748748800 + }, + { + "epoch": 0.23, + "learning_rate": 0.000390481444332999, + "loss": 2.8535, + "theoretical_loss": 3.753859162585226, + "tokens_seen": 748748800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904714142427282, + "loss": 2.8332, + "theoretical_loss": 3.7538262025187246, + "tokens_seen": 748814336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039046138415245734, + "loss": 2.6704, + "theoretical_loss": 3.753793246144374, + "tokens_seen": 748879872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039045135406218657, + "loss": 2.8584, + "theoretical_loss": 3.753760293461439, + "tokens_seen": 748945408 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039044132397191575, + "loss": 2.7273, + "theoretical_loss": 3.7537273444691825, + "tokens_seen": 749010944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039043129388164493, + "loss": 3.0141, + "theoretical_loss": 3.753694399166867, + "tokens_seen": 749076480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904212637913741, + "loss": 2.5281, + "theoretical_loss": 3.7536614575537586, + "tokens_seen": 749142016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904112337011033, + "loss": 2.7591, + "theoretical_loss": 3.75362851962912, + "tokens_seen": 749207552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003904012036108325, + "loss": 2.7426, + "theoretical_loss": 3.7535955853922167, + "tokens_seen": 749273088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903911735205617, + "loss": 2.9189, + "theoretical_loss": 3.7535626548423124, + "tokens_seen": 749338624 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903811434302909, + "loss": 2.7107, + "theoretical_loss": 3.7535297279786723, + "tokens_seen": 749404160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903711133400201, + "loss": 2.8125, + "theoretical_loss": 3.7534968048005615, + "tokens_seen": 749469696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039036108324974926, + "loss": 2.8113, + "theoretical_loss": 3.7534638853072453, + "tokens_seen": 749535232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039035105315947844, + "loss": 2.7766, + "theoretical_loss": 3.7534309694979884, + "tokens_seen": 749600768 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039034102306920767, + "loss": 2.8065, + "theoretical_loss": 3.753398057372058, + "tokens_seen": 749666304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903309929789368, + "loss": 2.6713, + "theoretical_loss": 3.7533651489287188, + "tokens_seen": 749731840 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039032096288866603, + "loss": 2.8039, + "theoretical_loss": 3.753332244167237, + "tokens_seen": 749797376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903109327983952, + "loss": 2.7021, + "theoretical_loss": 3.753299343086879, + "tokens_seen": 749862912 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003903009027081244, + "loss": 2.6358, + "theoretical_loss": 3.753266445686912, + "tokens_seen": 749928448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902908726178536, + "loss": 2.5634, + "theoretical_loss": 3.753233551966602, + "tokens_seen": 749993984 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039028084252758276, + "loss": 2.717, + "theoretical_loss": 3.7532006619252156, + "tokens_seen": 750059520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039027081243731194, + "loss": 2.8146, + "theoretical_loss": 3.753167775562021, + "tokens_seen": 750125056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902607823470412, + "loss": 2.799, + "theoretical_loss": 3.753134892876285, + "tokens_seen": 750190592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902507522567703, + "loss": 2.648, + "theoretical_loss": 3.753102013867275, + "tokens_seen": 750256128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039024072216649954, + "loss": 2.6237, + "theoretical_loss": 3.753069138534259, + "tokens_seen": 750321664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 421843, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.485619068145752, + "objective/train/theoretical_loss": 3.753036266876505, + "objective/train/tokens_used": 770847200, + "theoretical_loss": 3.753036266876505, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039023069207622866, + "loss": 2.5361, + "theoretical_loss": 3.753036266876505, + "tokens_seen": 750387200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902206619859579, + "loss": 2.828, + "theoretical_loss": 3.753003398893281, + "tokens_seen": 750452736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003902106318956871, + "loss": 2.6835, + "theoretical_loss": 3.7529705345838558, + "tokens_seen": 750518272 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039020060180541626, + "loss": 2.8157, + "theoretical_loss": 3.752937673947498, + "tokens_seen": 750583808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039019057171514544, + "loss": 2.6794, + "theoretical_loss": 3.7529048169834756, + "tokens_seen": 750649344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901805416248747, + "loss": 2.8133, + "theoretical_loss": 3.7528719636910584, + "tokens_seen": 750714880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901705115346038, + "loss": 2.6176, + "theoretical_loss": 3.7528391140695154, + "tokens_seen": 750780416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039016048144433304, + "loss": 2.9973, + "theoretical_loss": 3.7528062681181167, + "tokens_seen": 750845952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039015045135406217, + "loss": 2.8111, + "theoretical_loss": 3.752773425836131, + "tokens_seen": 750911488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901404212637914, + "loss": 2.8781, + "theoretical_loss": 3.7527405872228288, + "tokens_seen": 750977024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003901303911735206, + "loss": 2.8973, + "theoretical_loss": 3.75270775227748, + "tokens_seen": 751042560 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039012036108324977, + "loss": 2.7338, + "theoretical_loss": 3.7526749209993544, + "tokens_seen": 751108096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039011033099297895, + "loss": 2.7779, + "theoretical_loss": 3.752642093387723, + "tokens_seen": 751173632 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039010030090270813, + "loss": 2.7332, + "theoretical_loss": 3.752609269441857, + "tokens_seen": 751239168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900902708124373, + "loss": 2.6754, + "theoretical_loss": 3.7525764491610265, + "tokens_seen": 751304704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039008024072216654, + "loss": 2.7638, + "theoretical_loss": 3.7525436325445027, + "tokens_seen": 751370240 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039007021063189567, + "loss": 2.9505, + "theoretical_loss": 3.7525108195915573, + "tokens_seen": 751435776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900601805416249, + "loss": 2.5367, + "theoretical_loss": 3.752478010301462, + "tokens_seen": 751501312 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039005015045135403, + "loss": 2.679, + "theoretical_loss": 3.7524452046734877, + "tokens_seen": 751566848 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039004012036108327, + "loss": 2.8081, + "theoretical_loss": 3.752412402706907, + "tokens_seen": 751632384 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039003009027081245, + "loss": 2.8126, + "theoretical_loss": 3.7523796044009923, + "tokens_seen": 751697920 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039002006018054163, + "loss": 2.6605, + "theoretical_loss": 3.7523468097550152, + "tokens_seen": 751763456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003900100300902708, + "loss": 2.6298, + "theoretical_loss": 3.7523140187682493, + "tokens_seen": 751828992 + }, + { + "epoch": 0.23, + "learning_rate": 0.00039000000000000005, + "loss": 2.7272, + "theoretical_loss": 3.7522812314399663, + "tokens_seen": 751894528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899899699097292, + "loss": 2.9884, + "theoretical_loss": 3.7522484477694396, + "tokens_seen": 751960064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 423030, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.662358283996582, + "objective/train/theoretical_loss": 3.7522156677559426, + "objective/train/tokens_used": 772485600, + "theoretical_loss": 3.7522156677559426, + "tokens_seen": 752025600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899799398194584, + "loss": 2.7389, + "theoretical_loss": 3.7522156677559426, + "tokens_seen": 752025600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038996990972918754, + "loss": 2.9452, + "theoretical_loss": 3.752182891398749, + "tokens_seen": 752091136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038995987963891677, + "loss": 2.9664, + "theoretical_loss": 3.752150118697132, + "tokens_seen": 752156672 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038994984954864595, + "loss": 2.7687, + "theoretical_loss": 3.752117349650365, + "tokens_seen": 752222208 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038993981945837513, + "loss": 2.9713, + "theoretical_loss": 3.752084584257723, + "tokens_seen": 752287744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899297893681043, + "loss": 2.8341, + "theoretical_loss": 3.75205182251848, + "tokens_seen": 752353280 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899197592778335, + "loss": 2.7872, + "theoretical_loss": 3.7520190644319094, + "tokens_seen": 752418816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003899097291875627, + "loss": 2.5177, + "theoretical_loss": 3.751986309997287, + "tokens_seen": 752484352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898996990972919, + "loss": 3.0072, + "theoretical_loss": 3.751953559213888, + "tokens_seen": 752549888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038988966900702104, + "loss": 2.7622, + "theoretical_loss": 3.7519208120809857, + "tokens_seen": 752615424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898796389167503, + "loss": 2.8546, + "theoretical_loss": 3.7518880685978573, + "tokens_seen": 752680960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898696088264794, + "loss": 2.6212, + "theoretical_loss": 3.751855328763777, + "tokens_seen": 752746496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038985957873620864, + "loss": 2.4672, + "theoretical_loss": 3.751822592578022, + "tokens_seen": 752812032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898495486459378, + "loss": 2.624, + "theoretical_loss": 3.7517898600398665, + "tokens_seen": 752877568 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389839518555667, + "loss": 2.7594, + "theoretical_loss": 3.7517571311485867, + "tokens_seen": 752943104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898294884653962, + "loss": 2.8784, + "theoretical_loss": 3.7517244059034605, + "tokens_seen": 753008640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003898194583751254, + "loss": 2.57, + "theoretical_loss": 3.7516916843037627, + "tokens_seen": 753074176 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038980942828485454, + "loss": 2.69, + "theoretical_loss": 3.751658966348771, + "tokens_seen": 753139712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897993981945838, + "loss": 2.7796, + "theoretical_loss": 3.751626252037762, + "tokens_seen": 753205248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897893681043129, + "loss": 2.8479, + "theoretical_loss": 3.7515935413700134, + "tokens_seen": 753270784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038977933801404214, + "loss": 2.803, + "theoretical_loss": 3.7515608343448017, + "tokens_seen": 753336320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897693079237713, + "loss": 2.6135, + "theoretical_loss": 3.751528130961405, + "tokens_seen": 753401856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897592778335005, + "loss": 2.8919, + "theoretical_loss": 3.7514954312191007, + "tokens_seen": 753467392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897492477432297, + "loss": 2.759, + "theoretical_loss": 3.751462735117167, + "tokens_seen": 753532928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038973921765295886, + "loss": 2.639, + "theoretical_loss": 3.7514300426548823, + "tokens_seen": 753598464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 424346, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4410223960876465, + "objective/train/theoretical_loss": 3.751397353831524, + "objective/train/tokens_used": 774124000, + "theoretical_loss": 3.751397353831524, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038972918756268805, + "loss": 2.5345, + "theoretical_loss": 3.751397353831524, + "tokens_seen": 753664000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897191574724173, + "loss": 2.8886, + "theoretical_loss": 3.751364668646372, + "tokens_seen": 753729536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003897091273821464, + "loss": 2.5767, + "theoretical_loss": 3.7513319870987045, + "tokens_seen": 753795072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038969909729187564, + "loss": 2.6723, + "theoretical_loss": 3.7512993091878, + "tokens_seen": 753860608 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038968906720160477, + "loss": 2.8139, + "theoretical_loss": 3.751266634912939, + "tokens_seen": 753926144 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389679037111334, + "loss": 2.673, + "theoretical_loss": 3.7512339642734, + "tokens_seen": 753991680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896690070210632, + "loss": 2.4749, + "theoretical_loss": 3.7512012972684623, + "tokens_seen": 754057216 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038965897693079237, + "loss": 2.8392, + "theoretical_loss": 3.751168633897406, + "tokens_seen": 754122752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038964894684052155, + "loss": 2.6861, + "theoretical_loss": 3.751135974159511, + "tokens_seen": 754188288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003896389167502508, + "loss": 2.6624, + "theoretical_loss": 3.7511033180540583, + "tokens_seen": 754253824 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038962888665997997, + "loss": 2.7724, + "theoretical_loss": 3.7510706655803276, + "tokens_seen": 754319360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038961885656970915, + "loss": 2.9919, + "theoretical_loss": 3.7510380167375996, + "tokens_seen": 754384896 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038960882647943833, + "loss": 2.6663, + "theoretical_loss": 3.7510053715251557, + "tokens_seen": 754450432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895987963891675, + "loss": 2.778, + "theoretical_loss": 3.750972729942276, + "tokens_seen": 754515968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038958876629889674, + "loss": 2.795, + "theoretical_loss": 3.7509400919882427, + "tokens_seen": 754581504 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038957873620862587, + "loss": 2.6879, + "theoretical_loss": 3.7509074576623362, + "tokens_seen": 754647040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003895687061183551, + "loss": 2.8031, + "theoretical_loss": 3.7508748269638397, + "tokens_seen": 754712576 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038955867602808423, + "loss": 2.7347, + "theoretical_loss": 3.7508421998920336, + "tokens_seen": 754778112 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038954864593781347, + "loss": 2.8483, + "theoretical_loss": 3.7508095764462004, + "tokens_seen": 754843648 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038953861584754265, + "loss": 2.8203, + "theoretical_loss": 3.750776956625623, + "tokens_seen": 754909184 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038952858575727183, + "loss": 2.6643, + "theoretical_loss": 3.7507443404295833, + "tokens_seen": 754974720 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389518555667001, + "loss": 2.8899, + "theoretical_loss": 3.7507117278573636, + "tokens_seen": 755040256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038950852557673025, + "loss": 2.8196, + "theoretical_loss": 3.7506791189082476, + "tokens_seen": 755105792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894984954864594, + "loss": 2.7476, + "theoretical_loss": 3.750646513581518, + "tokens_seen": 755171328 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894884653961886, + "loss": 2.5687, + "theoretical_loss": 3.750613911876458, + "tokens_seen": 755236864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 425020, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5594170093536377, + "objective/train/theoretical_loss": 3.750581313792351, + "objective/train/tokens_used": 775762400, + "theoretical_loss": 3.750581313792351, + "tokens_seen": 755302400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038947843530591774, + "loss": 2.419, + "theoretical_loss": 3.750581313792351, + "tokens_seen": 755302400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038946840521564697, + "loss": 2.659, + "theoretical_loss": 3.7505487193284814, + "tokens_seen": 755367936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038945837512537615, + "loss": 2.7292, + "theoretical_loss": 3.750516128484133, + "tokens_seen": 755433472 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038944834503510533, + "loss": 2.8103, + "theoretical_loss": 3.7504835412585886, + "tokens_seen": 755499008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894383149448345, + "loss": 2.7047, + "theoretical_loss": 3.7504509576511342, + "tokens_seen": 755564544 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894282848545637, + "loss": 2.6846, + "theoretical_loss": 3.7504183776610533, + "tokens_seen": 755630080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894182547642929, + "loss": 2.7696, + "theoretical_loss": 3.7503858012876305, + "tokens_seen": 755695616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003894082246740221, + "loss": 2.746, + "theoretical_loss": 3.7503532285301517, + "tokens_seen": 755761152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038939819458375124, + "loss": 2.7698, + "theoretical_loss": 3.7503206593879015, + "tokens_seen": 755826688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893881644934805, + "loss": 2.661, + "theoretical_loss": 3.7502880938601653, + "tokens_seen": 755892224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893781344032096, + "loss": 2.5849, + "theoretical_loss": 3.750255531946228, + "tokens_seen": 755957760 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038936810431293884, + "loss": 2.7654, + "theoretical_loss": 3.7502229736453763, + "tokens_seen": 756023296 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389358074222668, + "loss": 2.8642, + "theoretical_loss": 3.7501904189568958, + "tokens_seen": 756088832 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893480441323972, + "loss": 2.7808, + "theoretical_loss": 3.7501578678800724, + "tokens_seen": 756154368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893380140421264, + "loss": 2.4493, + "theoretical_loss": 3.750125320414192, + "tokens_seen": 756219904 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003893279839518556, + "loss": 2.7259, + "theoretical_loss": 3.7500927765585423, + "tokens_seen": 756285440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038931795386158474, + "loss": 2.7156, + "theoretical_loss": 3.75006023631241, + "tokens_seen": 756350976 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389307923771314, + "loss": 2.7651, + "theoretical_loss": 3.7500276996750808, + "tokens_seen": 756416512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892978936810431, + "loss": 2.979, + "theoretical_loss": 3.749995166645843, + "tokens_seen": 756482048 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038928786359077234, + "loss": 3.0294, + "theoretical_loss": 3.749962637223984, + "tokens_seen": 756547584 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892778335005015, + "loss": 2.7881, + "theoretical_loss": 3.74993011140879, + "tokens_seen": 756613120 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892678034102307, + "loss": 2.6948, + "theoretical_loss": 3.7498975891995503, + "tokens_seen": 756678656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892577733199599, + "loss": 2.7056, + "theoretical_loss": 3.7498650705955523, + "tokens_seen": 756744192 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038924774322968906, + "loss": 2.6511, + "theoretical_loss": 3.7498325555960843, + "tokens_seen": 756809728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038923771313941825, + "loss": 2.7346, + "theoretical_loss": 3.749800044200434, + "tokens_seen": 756875264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 426245, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.891552448272705, + "objective/train/theoretical_loss": 3.749767536407891, + "objective/train/tokens_used": 777400800, + "theoretical_loss": 3.749767536407891, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892276830491475, + "loss": 2.7255, + "theoretical_loss": 3.749767536407891, + "tokens_seen": 756940800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003892176529588766, + "loss": 2.753, + "theoretical_loss": 3.7497350322177434, + "tokens_seen": 757006336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038920762286860584, + "loss": 2.7117, + "theoretical_loss": 3.7497025316292802, + "tokens_seen": 757071872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038919759277833497, + "loss": 2.6464, + "theoretical_loss": 3.749670034641791, + "tokens_seen": 757137408 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891875626880642, + "loss": 2.8426, + "theoretical_loss": 3.7496375412545646, + "tokens_seen": 757202944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891775325977934, + "loss": 2.6978, + "theoretical_loss": 3.749605051466891, + "tokens_seen": 757268480 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038916750250752257, + "loss": 2.936, + "theoretical_loss": 3.74957256527806, + "tokens_seen": 757334016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038915747241725175, + "loss": 2.8052, + "theoretical_loss": 3.7495400826873615, + "tokens_seen": 757399552 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389147442326981, + "loss": 2.7019, + "theoretical_loss": 3.7495076036940853, + "tokens_seen": 757465088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891374122367101, + "loss": 2.8646, + "theoretical_loss": 3.749475128297523, + "tokens_seen": 757530624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038912738214643935, + "loss": 2.6208, + "theoretical_loss": 3.7494426564969636, + "tokens_seen": 757596160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891173520561685, + "loss": 2.8859, + "theoretical_loss": 3.749410188291699, + "tokens_seen": 757661696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003891073219658977, + "loss": 2.7943, + "theoretical_loss": 3.74937772368102, + "tokens_seen": 757727232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890972918756269, + "loss": 2.7624, + "theoretical_loss": 3.7493452626642174, + "tokens_seen": 757792768 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038908726178535607, + "loss": 2.7132, + "theoretical_loss": 3.7493128052405824, + "tokens_seen": 757858304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038907723169508525, + "loss": 2.7807, + "theoretical_loss": 3.7492803514094075, + "tokens_seen": 757923840 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038906720160481443, + "loss": 2.6621, + "theoretical_loss": 3.7492479011699835, + "tokens_seen": 757989376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890571715145436, + "loss": 2.8097, + "theoretical_loss": 3.749215454521604, + "tokens_seen": 758054912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038904714142427285, + "loss": 2.7598, + "theoretical_loss": 3.7491830114635594, + "tokens_seen": 758120448 + }, + { + "epoch": 0.23, + "learning_rate": 0.000389037111334002, + "loss": 2.876, + "theoretical_loss": 3.749150571995143, + "tokens_seen": 758185984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890270812437312, + "loss": 2.8083, + "theoretical_loss": 3.749118136115647, + "tokens_seen": 758251520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890170511534604, + "loss": 2.6312, + "theoretical_loss": 3.749085703824364, + "tokens_seen": 758317056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003890070210631896, + "loss": 2.7658, + "theoretical_loss": 3.7490532751205885, + "tokens_seen": 758382592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038899699097291876, + "loss": 2.7504, + "theoretical_loss": 3.749020850003612, + "tokens_seen": 758448128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038898696088264794, + "loss": 2.6115, + "theoretical_loss": 3.7489884284727286, + "tokens_seen": 758513664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 426718, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.676023244857788, + "objective/train/theoretical_loss": 3.748956010527232, + "objective/train/tokens_used": 779039200, + "theoretical_loss": 3.748956010527232, + "tokens_seen": 758579200 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889769307923771, + "loss": 2.7646, + "theoretical_loss": 3.748956010527232, + "tokens_seen": 758579200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038896690070210635, + "loss": 2.8833, + "theoretical_loss": 3.7489235961664154, + "tokens_seen": 758644736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889568706118355, + "loss": 2.7432, + "theoretical_loss": 3.748891185389574, + "tokens_seen": 758710272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889468405215647, + "loss": 2.5897, + "theoretical_loss": 3.7488587781960008, + "tokens_seen": 758775808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038893681043129384, + "loss": 2.9097, + "theoretical_loss": 3.748826374584991, + "tokens_seen": 758841344 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003889267803410231, + "loss": 2.9009, + "theoretical_loss": 3.748793974555838, + "tokens_seen": 758906880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038891675025075226, + "loss": 2.6654, + "theoretical_loss": 3.748761578107838, + "tokens_seen": 758972416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038890672016048144, + "loss": 2.9037, + "theoretical_loss": 3.7487291852402853, + "tokens_seen": 759037952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888966900702106, + "loss": 2.5612, + "theoretical_loss": 3.7486967959524753, + "tokens_seen": 759103488 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888866599799398, + "loss": 2.5881, + "theoretical_loss": 3.7486644102437037, + "tokens_seen": 759169024 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038887662988966904, + "loss": 2.9656, + "theoretical_loss": 3.7486320281132657, + "tokens_seen": 759234560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888665997993982, + "loss": 2.801, + "theoretical_loss": 3.748599649560457, + "tokens_seen": 759300096 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888565697091274, + "loss": 2.7532, + "theoretical_loss": 3.7485672745845733, + "tokens_seen": 759365632 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888465396188566, + "loss": 2.4411, + "theoretical_loss": 3.748534903184912, + "tokens_seen": 759431168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888365095285858, + "loss": 2.7694, + "theoretical_loss": 3.7485025353607684, + "tokens_seen": 759496704 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038882647943831494, + "loss": 2.6058, + "theoretical_loss": 3.7484701711114394, + "tokens_seen": 759562240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888164493480442, + "loss": 2.8315, + "theoretical_loss": 3.748437810436222, + "tokens_seen": 759627776 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003888064192577733, + "loss": 2.9115, + "theoretical_loss": 3.748405453334413, + "tokens_seen": 759693312 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038879638916750254, + "loss": 2.6568, + "theoretical_loss": 3.7483730998053097, + "tokens_seen": 759758848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887863590772317, + "loss": 2.7921, + "theoretical_loss": 3.748340749848209, + "tokens_seen": 759824384 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887763289869609, + "loss": 2.8153, + "theoretical_loss": 3.74830840346241, + "tokens_seen": 759889920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887662988966901, + "loss": 2.6874, + "theoretical_loss": 3.7482760606472088, + "tokens_seen": 759955456 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038875626880641927, + "loss": 2.949, + "theoretical_loss": 3.748243721401904, + "tokens_seen": 760020992 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038874623871614845, + "loss": 2.771, + "theoretical_loss": 3.748211385725793, + "tokens_seen": 760086528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887362086258777, + "loss": 2.7016, + "theoretical_loss": 3.748179053618176, + "tokens_seen": 760152064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 427515, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.012765645980835, + "objective/train/theoretical_loss": 3.7481467250783504, + "objective/train/tokens_used": 780677600, + "theoretical_loss": 3.7481467250783504, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003887261785356068, + "loss": 3.079, + "theoretical_loss": 3.7481467250783504, + "tokens_seen": 760217600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038871614844533604, + "loss": 2.777, + "theoretical_loss": 3.748114400105615, + "tokens_seen": 760283136 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038870611835506517, + "loss": 2.8811, + "theoretical_loss": 3.748082078699269, + "tokens_seen": 760348672 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886960882647944, + "loss": 2.8731, + "theoretical_loss": 3.7480497608586116, + "tokens_seen": 760414208 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886860581745236, + "loss": 2.8526, + "theoretical_loss": 3.748017446582942, + "tokens_seen": 760479744 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038867602808425277, + "loss": 2.9608, + "theoretical_loss": 3.74798513587156, + "tokens_seen": 760545280 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038866599799398195, + "loss": 2.7168, + "theoretical_loss": 3.747952828723765, + "tokens_seen": 760610816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886559679037112, + "loss": 2.7147, + "theoretical_loss": 3.7479205251388574, + "tokens_seen": 760676352 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886459378134403, + "loss": 2.8606, + "theoretical_loss": 3.747888225116137, + "tokens_seen": 760741888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038863590772316955, + "loss": 2.9467, + "theoretical_loss": 3.7478559286549045, + "tokens_seen": 760807424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886258776328987, + "loss": 2.7865, + "theoretical_loss": 3.74782363575446, + "tokens_seen": 760872960 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886158475426279, + "loss": 2.7565, + "theoretical_loss": 3.747791346414105, + "tokens_seen": 760938496 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003886058174523571, + "loss": 2.7405, + "theoretical_loss": 3.74775906063314, + "tokens_seen": 761004032 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038859578736208627, + "loss": 2.7645, + "theoretical_loss": 3.747726778410866, + "tokens_seen": 761069568 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038858575727181545, + "loss": 2.9492, + "theoretical_loss": 3.7476944997465846, + "tokens_seen": 761135104 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038857572718154463, + "loss": 2.8445, + "theoretical_loss": 3.747662224639597, + "tokens_seen": 761200640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885656970912738, + "loss": 2.7768, + "theoretical_loss": 3.7476299530892057, + "tokens_seen": 761266176 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038855566700100305, + "loss": 3.0213, + "theoretical_loss": 3.7475976850947124, + "tokens_seen": 761331712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885456369107322, + "loss": 2.9531, + "theoretical_loss": 3.7475654206554188, + "tokens_seen": 761397248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885356068204614, + "loss": 2.9068, + "theoretical_loss": 3.7475331597706267, + "tokens_seen": 761462784 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885255767301906, + "loss": 2.7472, + "theoretical_loss": 3.74750090243964, + "tokens_seen": 761528320 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003885155466399198, + "loss": 2.8358, + "theoretical_loss": 3.747468648661761, + "tokens_seen": 761593856 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038850551654964896, + "loss": 3.0352, + "theoretical_loss": 3.7474363984362924, + "tokens_seen": 761659392 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038849548645937814, + "loss": 2.7928, + "theoretical_loss": 3.7474041517625376, + "tokens_seen": 761724928 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884854563691073, + "loss": 2.9632, + "theoretical_loss": 3.7473719086397996, + "tokens_seen": 761790464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 428729, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0259807109832764, + "objective/train/theoretical_loss": 3.747339669067382, + "objective/train/tokens_used": 782316000, + "theoretical_loss": 3.747339669067382, + "tokens_seen": 761856000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038847542627883655, + "loss": 3.03, + "theoretical_loss": 3.747339669067382, + "tokens_seen": 761856000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884653961885657, + "loss": 2.7308, + "theoretical_loss": 3.747307433044589, + "tokens_seen": 761921536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884553660982949, + "loss": 2.7257, + "theoretical_loss": 3.7472752005707233, + "tokens_seen": 761987072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038844533600802404, + "loss": 2.8627, + "theoretical_loss": 3.7472429716450897, + "tokens_seen": 762052608 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884353059177533, + "loss": 2.8747, + "theoretical_loss": 3.747210746266993, + "tokens_seen": 762118144 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038842527582748246, + "loss": 2.8288, + "theoretical_loss": 3.747178524435738, + "tokens_seen": 762183680 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038841524573721164, + "loss": 2.7417, + "theoretical_loss": 3.747146306150628, + "tokens_seen": 762249216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003884052156469408, + "loss": 2.8478, + "theoretical_loss": 3.7471140914109684, + "tokens_seen": 762314752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038839518555667, + "loss": 2.6023, + "theoretical_loss": 3.7470818802160646, + "tokens_seen": 762380288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883851554663992, + "loss": 2.88, + "theoretical_loss": 3.747049672565222, + "tokens_seen": 762445824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883751253761284, + "loss": 2.9135, + "theoretical_loss": 3.747017468457746, + "tokens_seen": 762511360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038836509528585755, + "loss": 3.1032, + "theoretical_loss": 3.7469852678929416, + "tokens_seen": 762576896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883550651955868, + "loss": 2.8413, + "theoretical_loss": 3.7469530708701155, + "tokens_seen": 762642432 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038834503510531596, + "loss": 2.6678, + "theoretical_loss": 3.7469208773885736, + "tokens_seen": 762707968 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038833500501504514, + "loss": 2.764, + "theoretical_loss": 3.7468886874476217, + "tokens_seen": 762773504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883249749247743, + "loss": 2.7965, + "theoretical_loss": 3.746856501046567, + "tokens_seen": 762839040 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883149448345035, + "loss": 2.6913, + "theoretical_loss": 3.746824318184715, + "tokens_seen": 762904576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003883049147442327, + "loss": 2.9067, + "theoretical_loss": 3.7467921388613745, + "tokens_seen": 762970112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882948846539619, + "loss": 2.7015, + "theoretical_loss": 3.746759963075851, + "tokens_seen": 763035648 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038828485456369105, + "loss": 2.962, + "theoretical_loss": 3.7467277908274514, + "tokens_seen": 763101184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882748244734203, + "loss": 3.0969, + "theoretical_loss": 3.7466956221154843, + "tokens_seen": 763166720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882647943831494, + "loss": 2.8899, + "theoretical_loss": 3.746663456939257, + "tokens_seen": 763232256 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038825476429287865, + "loss": 3.1046, + "theoretical_loss": 3.7466312952980774, + "tokens_seen": 763297792 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038824473420260783, + "loss": 2.7427, + "theoretical_loss": 3.746599137191253, + "tokens_seen": 763363328 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388234704112337, + "loss": 2.9909, + "theoretical_loss": 3.7465669826180923, + "tokens_seen": 763428864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 429385, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7105305194854736, + "objective/train/theoretical_loss": 3.746534831577904, + "objective/train/tokens_used": 783954400, + "theoretical_loss": 3.746534831577904, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003882246740220662, + "loss": 2.7427, + "theoretical_loss": 3.746534831577904, + "tokens_seen": 763494400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038821464393179537, + "loss": 2.726, + "theoretical_loss": 3.7465026840699958, + "tokens_seen": 763559936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038820461384152455, + "loss": 2.7427, + "theoretical_loss": 3.7464705400936777, + "tokens_seen": 763625472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881945837512538, + "loss": 2.7322, + "theoretical_loss": 3.746438399648258, + "tokens_seen": 763691008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881845536609829, + "loss": 2.996, + "theoretical_loss": 3.7464062627330463, + "tokens_seen": 763756544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038817452357071215, + "loss": 2.5489, + "theoretical_loss": 3.7463741293473514, + "tokens_seen": 763822080 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038816449348044133, + "loss": 2.7436, + "theoretical_loss": 3.7463419994904834, + "tokens_seen": 763887616 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881544633901705, + "loss": 2.758, + "theoretical_loss": 3.746309873161752, + "tokens_seen": 763953152 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881444332998997, + "loss": 2.8826, + "theoretical_loss": 3.7462777503604667, + "tokens_seen": 764018688 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881344032096289, + "loss": 2.7736, + "theoretical_loss": 3.746245631085938, + "tokens_seen": 764084224 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881243731193581, + "loss": 2.6549, + "theoretical_loss": 3.7462135153374767, + "tokens_seen": 764149760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003881143430290873, + "loss": 2.7209, + "theoretical_loss": 3.7461814031143925, + "tokens_seen": 764215296 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038810431293881647, + "loss": 2.7706, + "theoretical_loss": 3.746149294415997, + "tokens_seen": 764280832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038809428284854565, + "loss": 2.7119, + "theoretical_loss": 3.7461171892416, + "tokens_seen": 764346368 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038808425275827483, + "loss": 2.4525, + "theoretical_loss": 3.746085087590514, + "tokens_seen": 764411904 + }, + { + "epoch": 0.23, + "learning_rate": 0.000388074222668004, + "loss": 2.7598, + "theoretical_loss": 3.7460529894620493, + "tokens_seen": 764477440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038806419257773325, + "loss": 2.6697, + "theoretical_loss": 3.7460208948555183, + "tokens_seen": 764542976 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880541624874624, + "loss": 2.7454, + "theoretical_loss": 3.7459888037702314, + "tokens_seen": 764608512 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880441323971916, + "loss": 2.8053, + "theoretical_loss": 3.7459567162055016, + "tokens_seen": 764674048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003880341023069208, + "loss": 2.5966, + "theoretical_loss": 3.7459246321606408, + "tokens_seen": 764739584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038802407221665, + "loss": 2.8628, + "theoretical_loss": 3.7458925516349613, + "tokens_seen": 764805120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038801404212637916, + "loss": 2.7437, + "theoretical_loss": 3.745860474627775, + "tokens_seen": 764870656 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038800401203610834, + "loss": 2.7495, + "theoretical_loss": 3.7458284011383953, + "tokens_seen": 764936192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879939819458375, + "loss": 2.5588, + "theoretical_loss": 3.745796331166135, + "tokens_seen": 765001728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038798395185556675, + "loss": 2.6211, + "theoretical_loss": 3.745764264710307, + "tokens_seen": 765067264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 430652, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8052632808685303, + "objective/train/theoretical_loss": 3.7457322017702244, + "objective/train/tokens_used": 785592800, + "theoretical_loss": 3.7457322017702244, + "tokens_seen": 765132800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879739217652959, + "loss": 2.8717, + "theoretical_loss": 3.7457322017702244, + "tokens_seen": 765132800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879638916750251, + "loss": 2.8154, + "theoretical_loss": 3.7457001423452008, + "tokens_seen": 765198336 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038795386158475424, + "loss": 2.8259, + "theoretical_loss": 3.7456680864345495, + "tokens_seen": 765263872 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879438314944835, + "loss": 2.955, + "theoretical_loss": 3.745636034037585, + "tokens_seen": 765329408 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038793380140421266, + "loss": 2.7045, + "theoretical_loss": 3.745603985153621, + "tokens_seen": 765394944 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038792377131394184, + "loss": 2.7434, + "theoretical_loss": 3.745571939781972, + "tokens_seen": 765460480 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387913741223671, + "loss": 2.6924, + "theoretical_loss": 3.7455398979219514, + "tokens_seen": 765526016 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003879037111334002, + "loss": 2.7036, + "theoretical_loss": 3.745507859572875, + "tokens_seen": 765591552 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878936810431294, + "loss": 2.8034, + "theoretical_loss": 3.745475824734057, + "tokens_seen": 765657088 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878836509528586, + "loss": 2.9501, + "theoretical_loss": 3.745443793404813, + "tokens_seen": 765722624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038787362086258775, + "loss": 2.675, + "theoretical_loss": 3.745411765584457, + "tokens_seen": 765788160 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387863590772317, + "loss": 2.8542, + "theoretical_loss": 3.7453797412723056, + "tokens_seen": 765853696 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038785356068204616, + "loss": 2.7184, + "theoretical_loss": 3.7453477204676737, + "tokens_seen": 765919232 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038784353059177534, + "loss": 2.8863, + "theoretical_loss": 3.7453157031698776, + "tokens_seen": 765984768 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878335005015045, + "loss": 2.6537, + "theoretical_loss": 3.745283689378232, + "tokens_seen": 766050304 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878234704112337, + "loss": 2.8882, + "theoretical_loss": 3.7452516790920543, + "tokens_seen": 766115840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878134403209629, + "loss": 2.7967, + "theoretical_loss": 3.745219672310661, + "tokens_seen": 766181376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003878034102306921, + "loss": 2.8745, + "theoretical_loss": 3.745187669033368, + "tokens_seen": 766246912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038779338014042125, + "loss": 2.7758, + "theoretical_loss": 3.745155669259491, + "tokens_seen": 766312448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877833500501505, + "loss": 2.8668, + "theoretical_loss": 3.745123672988349, + "tokens_seen": 766377984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877733199598796, + "loss": 2.8476, + "theoretical_loss": 3.745091680219258, + "tokens_seen": 766443520 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038776328986960885, + "loss": 2.8759, + "theoretical_loss": 3.7450596909515355, + "tokens_seen": 766509056 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038775325977933803, + "loss": 3.0193, + "theoretical_loss": 3.7450277051844987, + "tokens_seen": 766574592 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877432296890672, + "loss": 2.7882, + "theoretical_loss": 3.7449957229174653, + "tokens_seen": 766640128 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003877331995987964, + "loss": 2.9495, + "theoretical_loss": 3.7449637441497536, + "tokens_seen": 766705664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 431055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6919522285461426, + "objective/train/theoretical_loss": 3.744931768880681, + "objective/train/tokens_used": 787231200, + "theoretical_loss": 3.744931768880681, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038772316950852557, + "loss": 2.92, + "theoretical_loss": 3.744931768880681, + "tokens_seen": 766771200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038771313941825475, + "loss": 2.7832, + "theoretical_loss": 3.744899797109567, + "tokens_seen": 766836736 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387703109327984, + "loss": 2.8811, + "theoretical_loss": 3.7448678288357287, + "tokens_seen": 766902272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876930792377131, + "loss": 2.921, + "theoretical_loss": 3.7448358640584853, + "tokens_seen": 766967808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038768304914744235, + "loss": 2.9364, + "theoretical_loss": 3.7448039027771554, + "tokens_seen": 767033344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038767301905717153, + "loss": 2.5438, + "theoretical_loss": 3.744771944991058, + "tokens_seen": 767098880 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876629889669007, + "loss": 2.7965, + "theoretical_loss": 3.7447399906995127, + "tokens_seen": 767164416 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876529588766299, + "loss": 2.8438, + "theoretical_loss": 3.7447080399018393, + "tokens_seen": 767229952 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876429287863591, + "loss": 2.9344, + "theoretical_loss": 3.7446760925973557, + "tokens_seen": 767295488 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038763289869608826, + "loss": 2.7249, + "theoretical_loss": 3.744644148785383, + "tokens_seen": 767361024 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876228686058175, + "loss": 2.8412, + "theoretical_loss": 3.7446122084652416, + "tokens_seen": 767426560 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003876128385155466, + "loss": 2.777, + "theoretical_loss": 3.7445802716362504, + "tokens_seen": 767492096 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038760280842527585, + "loss": 2.8745, + "theoretical_loss": 3.7445483382977303, + "tokens_seen": 767557632 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387592778335005, + "loss": 2.7448, + "theoretical_loss": 3.744516408449002, + "tokens_seen": 767623168 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875827482447342, + "loss": 2.9832, + "theoretical_loss": 3.744484482089386, + "tokens_seen": 767688704 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875727181544634, + "loss": 2.8086, + "theoretical_loss": 3.744452559218203, + "tokens_seen": 767754240 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875626880641926, + "loss": 2.6926, + "theoretical_loss": 3.7444206398347752, + "tokens_seen": 767819776 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038755265797392176, + "loss": 2.8013, + "theoretical_loss": 3.744388723938423, + "tokens_seen": 767885312 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387542627883651, + "loss": 2.776, + "theoretical_loss": 3.7443568115284673, + "tokens_seen": 767950848 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875325977933801, + "loss": 2.659, + "theoretical_loss": 3.744324902604231, + "tokens_seen": 768016384 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038752256770310936, + "loss": 2.7347, + "theoretical_loss": 3.7442929971650356, + "tokens_seen": 768081920 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875125376128385, + "loss": 2.7669, + "theoretical_loss": 3.744261095210203, + "tokens_seen": 768147456 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003875025075225677, + "loss": 2.8019, + "theoretical_loss": 3.744229196739055, + "tokens_seen": 768212992 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874924774322969, + "loss": 2.8041, + "theoretical_loss": 3.7441973017509147, + "tokens_seen": 768278528 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874824473420261, + "loss": 2.8703, + "theoretical_loss": 3.7441654102451047, + "tokens_seen": 768344064 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 432209, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0975613594055176, + "objective/train/theoretical_loss": 3.744133522220947, + "objective/train/tokens_used": 788869600, + "theoretical_loss": 3.744133522220947, + "tokens_seen": 768409600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038747241725175526, + "loss": 2.8819, + "theoretical_loss": 3.744133522220947, + "tokens_seen": 768409600 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038746238716148444, + "loss": 2.7959, + "theoretical_loss": 3.7441016376777663, + "tokens_seen": 768475136 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874523570712136, + "loss": 2.9047, + "theoretical_loss": 3.744069756614884, + "tokens_seen": 768540672 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038744232698094286, + "loss": 2.8923, + "theoretical_loss": 3.744037879031625, + "tokens_seen": 768606208 + }, + { + "epoch": 0.23, + "learning_rate": 0.000387432296890672, + "loss": 2.9207, + "theoretical_loss": 3.7440060049273116, + "tokens_seen": 768671744 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874222668004012, + "loss": 2.703, + "theoretical_loss": 3.7439741343012676, + "tokens_seen": 768737280 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038741223671013035, + "loss": 2.7158, + "theoretical_loss": 3.7439422671528177, + "tokens_seen": 768802816 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003874022066198596, + "loss": 2.7988, + "theoretical_loss": 3.7439104034812862, + "tokens_seen": 768868352 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038739217652958876, + "loss": 2.9013, + "theoretical_loss": 3.743878543285997, + "tokens_seen": 768933888 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038738214643931795, + "loss": 2.932, + "theoretical_loss": 3.7438466865662745, + "tokens_seen": 768999424 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873721163490472, + "loss": 2.9536, + "theoretical_loss": 3.7438148333214434, + "tokens_seen": 769064960 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038736208625877636, + "loss": 2.6854, + "theoretical_loss": 3.7437829835508287, + "tokens_seen": 769130496 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038735205616850554, + "loss": 2.749, + "theoretical_loss": 3.743751137253756, + "tokens_seen": 769196032 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873420260782347, + "loss": 2.4222, + "theoretical_loss": 3.7437192944295496, + "tokens_seen": 769261568 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873319959879639, + "loss": 2.9377, + "theoretical_loss": 3.743687455077536, + "tokens_seen": 769327104 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873219658976931, + "loss": 2.5288, + "theoretical_loss": 3.74365561919704, + "tokens_seen": 769392640 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003873119358074223, + "loss": 2.9269, + "theoretical_loss": 3.7436237867873876, + "tokens_seen": 769458176 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038730190571715145, + "loss": 3.2117, + "theoretical_loss": 3.743591957847905, + "tokens_seen": 769523712 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872918756268807, + "loss": 2.8716, + "theoretical_loss": 3.7435601323779184, + "tokens_seen": 769589248 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872818455366098, + "loss": 2.9267, + "theoretical_loss": 3.743528310376754, + "tokens_seen": 769654784 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038727181544633905, + "loss": 2.8562, + "theoretical_loss": 3.7434964918437394, + "tokens_seen": 769720320 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038726178535606823, + "loss": 2.617, + "theoretical_loss": 3.7434646767782, + "tokens_seen": 769785856 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872517552657974, + "loss": 2.8305, + "theoretical_loss": 3.7434328651794635, + "tokens_seen": 769851392 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872417251755266, + "loss": 2.8065, + "theoretical_loss": 3.743401057046857, + "tokens_seen": 769916928 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038723169508525577, + "loss": 2.7927, + "theoretical_loss": 3.7433692523797077, + "tokens_seen": 769982464 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 432880, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1428420543670654, + "objective/train/theoretical_loss": 3.743337451177343, + "objective/train/tokens_used": 790508000, + "theoretical_loss": 3.743337451177343, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038722166499498495, + "loss": 3.0053, + "theoretical_loss": 3.743337451177343, + "tokens_seen": 770048000 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872116349047142, + "loss": 2.8568, + "theoretical_loss": 3.743305653439091, + "tokens_seen": 770113536 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003872016048144433, + "loss": 2.941, + "theoretical_loss": 3.7432738591642796, + "tokens_seen": 770179072 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038719157472417255, + "loss": 2.8203, + "theoretical_loss": 3.7432420683522363, + "tokens_seen": 770244608 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038718154463390173, + "loss": 2.8672, + "theoretical_loss": 3.7432102810022903, + "tokens_seen": 770310144 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871715145436309, + "loss": 2.8655, + "theoretical_loss": 3.743178497113769, + "tokens_seen": 770375680 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871614844533601, + "loss": 2.7875, + "theoretical_loss": 3.7431467166860015, + "tokens_seen": 770441216 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871514543630893, + "loss": 2.7909, + "theoretical_loss": 3.7431149397183177, + "tokens_seen": 770506752 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038714142427281846, + "loss": 2.8959, + "theoretical_loss": 3.743083166210045, + "tokens_seen": 770572288 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871313941825477, + "loss": 2.7932, + "theoretical_loss": 3.7430513961605136, + "tokens_seen": 770637824 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871213640922768, + "loss": 2.814, + "theoretical_loss": 3.7430196295690523, + "tokens_seen": 770703360 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038711133400200605, + "loss": 3.0201, + "theoretical_loss": 3.7429878664349916, + "tokens_seen": 770768896 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003871013039117352, + "loss": 2.8408, + "theoretical_loss": 3.74295610675766, + "tokens_seen": 770834432 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870912738214644, + "loss": 2.6841, + "theoretical_loss": 3.7429243505363887, + "tokens_seen": 770899968 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870812437311936, + "loss": 2.7229, + "theoretical_loss": 3.742892597770507, + "tokens_seen": 770965504 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870712136409228, + "loss": 2.8088, + "theoretical_loss": 3.7428608484593457, + "tokens_seen": 771031040 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038706118355065196, + "loss": 2.6387, + "theoretical_loss": 3.7428291026022356, + "tokens_seen": 771096576 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870511534603812, + "loss": 2.7468, + "theoretical_loss": 3.742797360198507, + "tokens_seen": 771162112 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870411233701103, + "loss": 2.7474, + "theoretical_loss": 3.7427656212474902, + "tokens_seen": 771227648 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038703109327983956, + "loss": 2.8616, + "theoretical_loss": 3.7427338857485175, + "tokens_seen": 771293184 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870210631895687, + "loss": 2.6703, + "theoretical_loss": 3.742702153700919, + "tokens_seen": 771358720 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870110330992979, + "loss": 2.7043, + "theoretical_loss": 3.742670425104027, + "tokens_seen": 771424256 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003870010030090271, + "loss": 2.797, + "theoretical_loss": 3.7426386999571735, + "tokens_seen": 771489792 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869909729187563, + "loss": 2.8079, + "theoretical_loss": 3.7426069782596887, + "tokens_seen": 771555328 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038698094282848546, + "loss": 2.8836, + "theoretical_loss": 3.7425752600109066, + "tokens_seen": 771620864 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 434052, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.293273448944092, + "objective/train/theoretical_loss": 3.742543545210158, + "objective/train/tokens_used": 792146400, + "theoretical_loss": 3.742543545210158, + "tokens_seen": 771686400 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038697091273821464, + "loss": 2.6631, + "theoretical_loss": 3.742543545210158, + "tokens_seen": 771686400 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869608826479438, + "loss": 2.892, + "theoretical_loss": 3.7425118338567755, + "tokens_seen": 771751936 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038695085255767306, + "loss": 2.8685, + "theoretical_loss": 3.7424801259500926, + "tokens_seen": 771817472 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869408224674022, + "loss": 2.8693, + "theoretical_loss": 3.7424484214894407, + "tokens_seen": 771883008 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869307923771314, + "loss": 2.8568, + "theoretical_loss": 3.7424167204741536, + "tokens_seen": 771948544 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038692076228686055, + "loss": 2.937, + "theoretical_loss": 3.7423850229035645, + "tokens_seen": 772014080 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003869107321965898, + "loss": 2.6878, + "theoretical_loss": 3.742353328777006, + "tokens_seen": 772079616 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038690070210631896, + "loss": 2.7514, + "theoretical_loss": 3.742321638093813, + "tokens_seen": 772145152 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038689067201604815, + "loss": 2.8053, + "theoretical_loss": 3.7422899508533174, + "tokens_seen": 772210688 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038688064192577733, + "loss": 2.6689, + "theoretical_loss": 3.7422582670548543, + "tokens_seen": 772276224 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038687061183550656, + "loss": 2.6903, + "theoretical_loss": 3.7422265866977575, + "tokens_seen": 772341760 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868605817452357, + "loss": 2.7236, + "theoretical_loss": 3.742194909781361, + "tokens_seen": 772407296 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868505516549649, + "loss": 2.9353, + "theoretical_loss": 3.7421632363049993, + "tokens_seen": 772472832 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038684052156469405, + "loss": 2.823, + "theoretical_loss": 3.7421315662680072, + "tokens_seen": 772538368 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003868304914744233, + "loss": 2.7551, + "theoretical_loss": 3.7420998996697197, + "tokens_seen": 772603904 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038682046138415247, + "loss": 2.9628, + "theoretical_loss": 3.7420682365094713, + "tokens_seen": 772669440 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038681043129388165, + "loss": 2.7756, + "theoretical_loss": 3.7420365767865977, + "tokens_seen": 772734976 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038680040120361083, + "loss": 2.9, + "theoretical_loss": 3.7420049205004338, + "tokens_seen": 772800512 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038679037111334, + "loss": 2.8719, + "theoretical_loss": 3.741973267650315, + "tokens_seen": 772866048 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867803410230692, + "loss": 2.9457, + "theoretical_loss": 3.741941618235577, + "tokens_seen": 772931584 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038677031093279843, + "loss": 2.8025, + "theoretical_loss": 3.7419099722555567, + "tokens_seen": 772997120 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038676028084252755, + "loss": 2.6723, + "theoretical_loss": 3.7418783297095892, + "tokens_seen": 773062656 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867502507522568, + "loss": 2.8905, + "theoretical_loss": 3.741846690597012, + "tokens_seen": 773128192 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867402206619859, + "loss": 2.7707, + "theoretical_loss": 3.7418150549171596, + "tokens_seen": 773193728 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038673019057171515, + "loss": 2.873, + "theoretical_loss": 3.74178342266937, + "tokens_seen": 773259264 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 434666, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.016434907913208, + "objective/train/theoretical_loss": 3.74175179385298, + "objective/train/tokens_used": 793784800, + "theoretical_loss": 3.74175179385298, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038672016048144433, + "loss": 2.8806, + "theoretical_loss": 3.74175179385298, + "tokens_seen": 773324800 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867101303911735, + "loss": 2.7735, + "theoretical_loss": 3.7417201684673262, + "tokens_seen": 773390336 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003867001003009027, + "loss": 2.8635, + "theoretical_loss": 3.741688546511746, + "tokens_seen": 773455872 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038669007021063193, + "loss": 2.7011, + "theoretical_loss": 3.7416569279855767, + "tokens_seen": 773521408 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038668004012036106, + "loss": 2.6612, + "theoretical_loss": 3.7416253128881563, + "tokens_seen": 773586944 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866700100300903, + "loss": 3.141, + "theoretical_loss": 3.7415937012188216, + "tokens_seen": 773652480 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866599799398194, + "loss": 2.8049, + "theoretical_loss": 3.7415620929769116, + "tokens_seen": 773718016 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038664994984954866, + "loss": 2.7794, + "theoretical_loss": 3.7415304881617644, + "tokens_seen": 773783552 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038663991975927784, + "loss": 2.9484, + "theoretical_loss": 3.7414988867727166, + "tokens_seen": 773849088 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386629889669007, + "loss": 2.8271, + "theoretical_loss": 3.741467288809109, + "tokens_seen": 773914624 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038661985957873625, + "loss": 2.9336, + "theoretical_loss": 3.7414356942702787, + "tokens_seen": 773980160 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003866098294884654, + "loss": 2.8063, + "theoretical_loss": 3.7414041031555656, + "tokens_seen": 774045696 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865997993981946, + "loss": 2.7585, + "theoretical_loss": 3.7413725154643083, + "tokens_seen": 774111232 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865897693079238, + "loss": 2.7239, + "theoretical_loss": 3.741340931195846, + "tokens_seen": 774176768 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386579739217653, + "loss": 2.6134, + "theoretical_loss": 3.7413093503495176, + "tokens_seen": 774242304 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038656970912738216, + "loss": 2.9152, + "theoretical_loss": 3.741277772924663, + "tokens_seen": 774307840 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865596790371114, + "loss": 2.7841, + "theoretical_loss": 3.7412461989206225, + "tokens_seen": 774373376 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865496489468405, + "loss": 2.7092, + "theoretical_loss": 3.741214628336736, + "tokens_seen": 774438912 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038653961885656976, + "loss": 2.724, + "theoretical_loss": 3.741183061172343, + "tokens_seen": 774504448 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865295887662989, + "loss": 2.8625, + "theoretical_loss": 3.7411514974267845, + "tokens_seen": 774569984 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865195586760281, + "loss": 3.0894, + "theoretical_loss": 3.7411199370994, + "tokens_seen": 774635520 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003865095285857573, + "loss": 2.8484, + "theoretical_loss": 3.741088380189532, + "tokens_seen": 774701056 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864994984954865, + "loss": 2.7995, + "theoretical_loss": 3.7410568266965196, + "tokens_seen": 774766592 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038648946840521566, + "loss": 2.8275, + "theoretical_loss": 3.7410252766197045, + "tokens_seen": 774832128 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038647943831494484, + "loss": 2.7769, + "theoretical_loss": 3.740993729958428, + "tokens_seen": 774897664 + }, + { + "epoch": 0.23, + "objective/train/docs_used": 435763, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.750361680984497, + "objective/train/theoretical_loss": 3.7409621867120313, + "objective/train/tokens_used": 795423200, + "theoretical_loss": 3.7409621867120313, + "tokens_seen": 774963200 + }, + { + "epoch": 0.23, + "learning_rate": 0.000386469408224674, + "loss": 2.6144, + "theoretical_loss": 3.7409621867120313, + "tokens_seen": 774963200 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038645937813440326, + "loss": 2.5263, + "theoretical_loss": 3.7409306468798564, + "tokens_seen": 775028736 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864493480441324, + "loss": 2.7634, + "theoretical_loss": 3.7408991104612452, + "tokens_seen": 775094272 + }, + { + "epoch": 0.23, + "learning_rate": 0.0003864393179538616, + "loss": 2.7988, + "theoretical_loss": 3.740867577455539, + "tokens_seen": 775159808 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038642928786359075, + "loss": 2.9711, + "theoretical_loss": 3.7408360478620803, + "tokens_seen": 775225344 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038641925777332, + "loss": 2.7774, + "theoretical_loss": 3.7408045216802117, + "tokens_seen": 775290880 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038640922768304917, + "loss": 2.8231, + "theoretical_loss": 3.740772998909275, + "tokens_seen": 775356416 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038639919759277835, + "loss": 2.9609, + "theoretical_loss": 3.740741479548613, + "tokens_seen": 775421952 + }, + { + "epoch": 0.23, + "learning_rate": 0.00038638916750250753, + "loss": 2.8501, + "theoretical_loss": 3.74070996359757, + "tokens_seen": 775487488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038637913741223676, + "loss": 2.7855, + "theoretical_loss": 3.7406784510554876, + "tokens_seen": 775553024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863691073219659, + "loss": 2.5942, + "theoretical_loss": 3.7406469419217094, + "tokens_seen": 775618560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863590772316951, + "loss": 2.7587, + "theoretical_loss": 3.7406154361955792, + "tokens_seen": 775684096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038634904714142425, + "loss": 2.4622, + "theoretical_loss": 3.7405839338764397, + "tokens_seen": 775749632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003863390170511535, + "loss": 2.7932, + "theoretical_loss": 3.740552434963636, + "tokens_seen": 775815168 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038632898696088267, + "loss": 2.8275, + "theoretical_loss": 3.740520939456511, + "tokens_seen": 775880704 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038631895687061185, + "loss": 2.9642, + "theoretical_loss": 3.740489447354409, + "tokens_seen": 775946240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038630892678034103, + "loss": 2.7347, + "theoretical_loss": 3.7404579586566746, + "tokens_seen": 776011776 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862988966900702, + "loss": 2.6956, + "theoretical_loss": 3.7404264733626524, + "tokens_seen": 776077312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862888665997994, + "loss": 3.0187, + "theoretical_loss": 3.740394991471687, + "tokens_seen": 776142848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038627883650952863, + "loss": 2.956, + "theoretical_loss": 3.740363512983123, + "tokens_seen": 776208384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038626880641925775, + "loss": 2.7489, + "theoretical_loss": 3.740332037896306, + "tokens_seen": 776273920 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386258776328987, + "loss": 2.8328, + "theoretical_loss": 3.740300566210581, + "tokens_seen": 776339456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862487462387161, + "loss": 3.0334, + "theoretical_loss": 3.7402690979252937, + "tokens_seen": 776404992 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038623871614844535, + "loss": 2.8642, + "theoretical_loss": 3.740237633039789, + "tokens_seen": 776470528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038622868605817453, + "loss": 2.7713, + "theoretical_loss": 3.7402061715534134, + "tokens_seen": 776536064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 436694, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.752096652984619, + "objective/train/theoretical_loss": 3.740174713465512, + "objective/train/tokens_used": 797061600, + "theoretical_loss": 3.740174713465512, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862186559679037, + "loss": 2.7836, + "theoretical_loss": 3.740174713465512, + "tokens_seen": 776601600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003862086258776329, + "loss": 2.7319, + "theoretical_loss": 3.7401432587754324, + "tokens_seen": 776667136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038619859578736213, + "loss": 2.9838, + "theoretical_loss": 3.7401118074825197, + "tokens_seen": 776732672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038618856569709126, + "loss": 2.8541, + "theoretical_loss": 3.7400803595861203, + "tokens_seen": 776798208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861785356068205, + "loss": 2.7572, + "theoretical_loss": 3.740048915085582, + "tokens_seen": 776863744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861685055165496, + "loss": 3.0645, + "theoretical_loss": 3.7400174739802505, + "tokens_seen": 776929280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038615847542627886, + "loss": 2.8094, + "theoretical_loss": 3.7399860362694737, + "tokens_seen": 776994816 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038614844533600804, + "loss": 2.7469, + "theoretical_loss": 3.7399546019525984, + "tokens_seen": 777060352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861384152457372, + "loss": 2.7164, + "theoretical_loss": 3.7399231710289724, + "tokens_seen": 777125888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861283851554664, + "loss": 2.7987, + "theoretical_loss": 3.7398917434979424, + "tokens_seen": 777191424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003861183550651956, + "loss": 2.9797, + "theoretical_loss": 3.7398603193588578, + "tokens_seen": 777256960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038610832497492476, + "loss": 2.8151, + "theoretical_loss": 3.739828898611065, + "tokens_seen": 777322496 + }, + { + "epoch": 0.24, + "learning_rate": 0.000386098294884654, + "loss": 2.8161, + "theoretical_loss": 3.7397974812539125, + "tokens_seen": 777388032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860882647943831, + "loss": 2.7239, + "theoretical_loss": 3.739766067286749, + "tokens_seen": 777453568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038607823470411236, + "loss": 2.7941, + "theoretical_loss": 3.739734656708923, + "tokens_seen": 777519104 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860682046138415, + "loss": 2.7307, + "theoretical_loss": 3.739703249519783, + "tokens_seen": 777584640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860581745235707, + "loss": 2.7405, + "theoretical_loss": 3.7396718457186777, + "tokens_seen": 777650176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860481444332999, + "loss": 2.8985, + "theoretical_loss": 3.739640445304957, + "tokens_seen": 777715712 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860381143430291, + "loss": 2.7029, + "theoretical_loss": 3.7396090482779685, + "tokens_seen": 777781248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038602808425275826, + "loss": 2.7713, + "theoretical_loss": 3.7395776546370634, + "tokens_seen": 777846784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860180541624875, + "loss": 3.0061, + "theoretical_loss": 3.7395462643815893, + "tokens_seen": 777912320 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003860080240722166, + "loss": 2.7808, + "theoretical_loss": 3.739514877510898, + "tokens_seen": 777977856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038599799398194586, + "loss": 2.9553, + "theoretical_loss": 3.739483494024338, + "tokens_seen": 778043392 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385987963891675, + "loss": 2.9095, + "theoretical_loss": 3.73945211392126, + "tokens_seen": 778108928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859779338014042, + "loss": 2.862, + "theoretical_loss": 3.739420737201015, + "tokens_seen": 778174464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 437315, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8093600273132324, + "objective/train/theoretical_loss": 3.7393893638629514, + "objective/train/tokens_used": 798700000, + "theoretical_loss": 3.7393893638629514, + "tokens_seen": 778240000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859679037111334, + "loss": 2.887, + "theoretical_loss": 3.7393893638629514, + "tokens_seen": 778240000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859578736208626, + "loss": 2.8717, + "theoretical_loss": 3.739357993906422, + "tokens_seen": 778305536 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038594784353059177, + "loss": 2.6532, + "theoretical_loss": 3.739326627330777, + "tokens_seen": 778371072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038593781344032095, + "loss": 2.8611, + "theoretical_loss": 3.7392952641353663, + "tokens_seen": 778436608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038592778335005013, + "loss": 2.6005, + "theoretical_loss": 3.739263904319543, + "tokens_seen": 778502144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038591775325977937, + "loss": 2.7977, + "theoretical_loss": 3.739232547882657, + "tokens_seen": 778567680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003859077231695085, + "loss": 2.921, + "theoretical_loss": 3.73920119482406, + "tokens_seen": 778633216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038589769307923773, + "loss": 2.6604, + "theoretical_loss": 3.7391698451431044, + "tokens_seen": 778698752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038588766298896685, + "loss": 2.8638, + "theoretical_loss": 3.7391384988391416, + "tokens_seen": 778764288 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858776328986961, + "loss": 3.0366, + "theoretical_loss": 3.7391071559115243, + "tokens_seen": 778829824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858676028084253, + "loss": 2.8517, + "theoretical_loss": 3.7390758163596036, + "tokens_seen": 778895360 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038585757271815445, + "loss": 2.8778, + "theoretical_loss": 3.739044480182733, + "tokens_seen": 778960896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858475426278837, + "loss": 2.49, + "theoretical_loss": 3.7390131473802652, + "tokens_seen": 779026432 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038583751253761287, + "loss": 2.9749, + "theoretical_loss": 3.7389818179515517, + "tokens_seen": 779091968 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038582748244734205, + "loss": 2.7421, + "theoretical_loss": 3.738950491895947, + "tokens_seen": 779157504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038581745235707123, + "loss": 2.663, + "theoretical_loss": 3.738919169212804, + "tokens_seen": 779223040 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003858074222668004, + "loss": 2.7339, + "theoretical_loss": 3.7388878499014746, + "tokens_seen": 779288576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857973921765296, + "loss": 2.6934, + "theoretical_loss": 3.7388565339613145, + "tokens_seen": 779354112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038578736208625883, + "loss": 2.7529, + "theoretical_loss": 3.738825221391676, + "tokens_seen": 779419648 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038577733199598796, + "loss": 2.9809, + "theoretical_loss": 3.738793912191913, + "tokens_seen": 779485184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857673019057172, + "loss": 2.6457, + "theoretical_loss": 3.7387626063613797, + "tokens_seen": 779550720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857572718154463, + "loss": 2.8474, + "theoretical_loss": 3.7387313038994305, + "tokens_seen": 779616256 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038574724172517555, + "loss": 2.6877, + "theoretical_loss": 3.7387000048054198, + "tokens_seen": 779681792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038573721163490473, + "loss": 2.9636, + "theoretical_loss": 3.738668709078702, + "tokens_seen": 779747328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857271815446339, + "loss": 2.7967, + "theoretical_loss": 3.7386374167186327, + "tokens_seen": 779812864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 438429, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.841843843460083, + "objective/train/theoretical_loss": 3.7386061277245655, + "objective/train/tokens_used": 800338400, + "theoretical_loss": 3.7386061277245655, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003857171514543631, + "loss": 3.0358, + "theoretical_loss": 3.7386061277245655, + "tokens_seen": 779878400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038570712136409233, + "loss": 3.0418, + "theoretical_loss": 3.7385748420958564, + "tokens_seen": 779943936 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038569709127382146, + "loss": 2.6862, + "theoretical_loss": 3.7385435598318604, + "tokens_seen": 780009472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856870611835507, + "loss": 2.7656, + "theoretical_loss": 3.738512280931933, + "tokens_seen": 780075008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856770310932798, + "loss": 2.7398, + "theoretical_loss": 3.73848100539543, + "tokens_seen": 780140544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038566700100300906, + "loss": 2.5016, + "theoretical_loss": 3.7384497332217075, + "tokens_seen": 780206080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038565697091273824, + "loss": 2.6233, + "theoretical_loss": 3.7384184644101204, + "tokens_seen": 780271616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856469408224674, + "loss": 2.7823, + "theoretical_loss": 3.7383871989600257, + "tokens_seen": 780337152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856369107321966, + "loss": 2.7999, + "theoretical_loss": 3.7383559368707804, + "tokens_seen": 780402688 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856268806419258, + "loss": 2.6636, + "theoretical_loss": 3.7383246781417396, + "tokens_seen": 780468224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038561685055165496, + "loss": 2.7589, + "theoretical_loss": 3.738293422772261, + "tokens_seen": 780533760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003856068204613842, + "loss": 2.6067, + "theoretical_loss": 3.738262170761701, + "tokens_seen": 780599296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855967903711133, + "loss": 3.0154, + "theoretical_loss": 3.738230922109417, + "tokens_seen": 780664832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038558676028084256, + "loss": 2.6455, + "theoretical_loss": 3.7381996768147663, + "tokens_seen": 780730368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855767301905717, + "loss": 2.9596, + "theoretical_loss": 3.738168434877106, + "tokens_seen": 780795904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855667001003009, + "loss": 2.868, + "theoretical_loss": 3.738137196295794, + "tokens_seen": 780861440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855566700100301, + "loss": 2.8184, + "theoretical_loss": 3.738105961070188, + "tokens_seen": 780926976 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855466399197593, + "loss": 2.8756, + "theoretical_loss": 3.7380747291996457, + "tokens_seen": 780992512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038553660982948846, + "loss": 2.7564, + "theoretical_loss": 3.738043500683526, + "tokens_seen": 781058048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855265797392177, + "loss": 2.8796, + "theoretical_loss": 3.7380122755211858, + "tokens_seen": 781123584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003855165496489468, + "loss": 2.8602, + "theoretical_loss": 3.7379810537119846, + "tokens_seen": 781189120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038550651955867606, + "loss": 2.7505, + "theoretical_loss": 3.7379498352552814, + "tokens_seen": 781254656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854964894684052, + "loss": 2.3087, + "theoretical_loss": 3.7379186201504346, + "tokens_seen": 781320192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854864593781344, + "loss": 2.7144, + "theoretical_loss": 3.737887408396803, + "tokens_seen": 781385728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854764292878636, + "loss": 2.4728, + "theoretical_loss": 3.737856199993746, + "tokens_seen": 781451264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 439045, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9254188537597656, + "objective/train/theoretical_loss": 3.737824994940622, + "objective/train/tokens_used": 801976800, + "theoretical_loss": 3.737824994940622, + "tokens_seen": 781516800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854663991975928, + "loss": 2.8778, + "theoretical_loss": 3.737824994940622, + "tokens_seen": 781516800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038545636910732197, + "loss": 2.9068, + "theoretical_loss": 3.7377937932367926, + "tokens_seen": 781582336 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038544633901705115, + "loss": 2.5584, + "theoretical_loss": 3.737762594881616, + "tokens_seen": 781647872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038543630892678033, + "loss": 2.6853, + "theoretical_loss": 3.7377313998744524, + "tokens_seen": 781713408 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038542627883650957, + "loss": 2.6201, + "theoretical_loss": 3.7377002082146618, + "tokens_seen": 781778944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003854162487462387, + "loss": 2.8869, + "theoretical_loss": 3.737669019901605, + "tokens_seen": 781844480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038540621865596793, + "loss": 2.7393, + "theoretical_loss": 3.7376378349346417, + "tokens_seen": 781910016 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038539618856569705, + "loss": 2.868, + "theoretical_loss": 3.737606653313133, + "tokens_seen": 781975552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853861584754263, + "loss": 2.9731, + "theoretical_loss": 3.737575475036439, + "tokens_seen": 782041088 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038537612838515547, + "loss": 2.7581, + "theoretical_loss": 3.737544300103922, + "tokens_seen": 782106624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038536609829488465, + "loss": 2.7906, + "theoretical_loss": 3.7375131285149417, + "tokens_seen": 782172160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038535606820461383, + "loss": 2.7958, + "theoretical_loss": 3.73748196026886, + "tokens_seen": 782237696 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038534603811434307, + "loss": 2.7489, + "theoretical_loss": 3.7374507953650387, + "tokens_seen": 782303232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853360080240722, + "loss": 2.8457, + "theoretical_loss": 3.7374196338028387, + "tokens_seen": 782368768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038532597793380143, + "loss": 2.6944, + "theoretical_loss": 3.7373884755816222, + "tokens_seen": 782434304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038531594784353056, + "loss": 2.95, + "theoretical_loss": 3.7373573207007516, + "tokens_seen": 782499840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003853059177532598, + "loss": 2.8403, + "theoretical_loss": 3.737326169159589, + "tokens_seen": 782565376 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385295887662989, + "loss": 2.7555, + "theoretical_loss": 3.7372950209574958, + "tokens_seen": 782630912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038528585757271816, + "loss": 2.7885, + "theoretical_loss": 3.7372638760938353, + "tokens_seen": 782696448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038527582748244734, + "loss": 2.7648, + "theoretical_loss": 3.7372327345679706, + "tokens_seen": 782761984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852657973921765, + "loss": 3.0226, + "theoretical_loss": 3.737201596379263, + "tokens_seen": 782827520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852557673019057, + "loss": 2.7361, + "theoretical_loss": 3.7371704615270778, + "tokens_seen": 782893056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038524573721163493, + "loss": 2.8869, + "theoretical_loss": 3.7371393300107765, + "tokens_seen": 782958592 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038523570712136406, + "loss": 2.7412, + "theoretical_loss": 3.7371082018297233, + "tokens_seen": 783024128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852256770310933, + "loss": 2.7908, + "theoretical_loss": 3.7370770769832813, + "tokens_seen": 783089664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 440183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6950714588165283, + "objective/train/theoretical_loss": 3.7370459554708146, + "objective/train/tokens_used": 803615200, + "theoretical_loss": 3.7370459554708146, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003852156469408225, + "loss": 2.6978, + "theoretical_loss": 3.7370459554708146, + "tokens_seen": 783155200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038520561685055166, + "loss": 2.9029, + "theoretical_loss": 3.737014837291687, + "tokens_seen": 783220736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038519558676028084, + "loss": 2.8232, + "theoretical_loss": 3.736983722445263, + "tokens_seen": 783286272 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038518555667001, + "loss": 2.7811, + "theoretical_loss": 3.7369526109309064, + "tokens_seen": 783351808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851755265797392, + "loss": 2.783, + "theoretical_loss": 3.7369215027479816, + "tokens_seen": 783417344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038516549648946844, + "loss": 2.7497, + "theoretical_loss": 3.736890397895853, + "tokens_seen": 783482880 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038515546639919756, + "loss": 2.7636, + "theoretical_loss": 3.7368592963738863, + "tokens_seen": 783548416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851454363089268, + "loss": 2.6221, + "theoretical_loss": 3.736828198181446, + "tokens_seen": 783613952 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851354062186559, + "loss": 3.0129, + "theoretical_loss": 3.736797103317897, + "tokens_seen": 783679488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038512537612838516, + "loss": 2.7719, + "theoretical_loss": 3.7367660117826054, + "tokens_seen": 783745024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851153460381144, + "loss": 2.8759, + "theoretical_loss": 3.736734923574935, + "tokens_seen": 783810560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003851053159478435, + "loss": 2.8192, + "theoretical_loss": 3.736703838694254, + "tokens_seen": 783876096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038509528585757276, + "loss": 2.7093, + "theoretical_loss": 3.7366727571399254, + "tokens_seen": 783941632 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850852557673019, + "loss": 2.8386, + "theoretical_loss": 3.736641678911318, + "tokens_seen": 784007168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850752256770311, + "loss": 2.8519, + "theoretical_loss": 3.7366106040077955, + "tokens_seen": 784072704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850651955867603, + "loss": 2.9498, + "theoretical_loss": 3.736579532428726, + "tokens_seen": 784138240 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850551654964895, + "loss": 2.9198, + "theoretical_loss": 3.7365484641734756, + "tokens_seen": 784203776 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038504513540621866, + "loss": 2.7531, + "theoretical_loss": 3.7365173992414107, + "tokens_seen": 784269312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850351053159479, + "loss": 2.6193, + "theoretical_loss": 3.7364863376318986, + "tokens_seen": 784334848 + }, + { + "epoch": 0.24, + "learning_rate": 0.000385025075225677, + "loss": 3.0243, + "theoretical_loss": 3.736455279344306, + "tokens_seen": 784400384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038501504513540626, + "loss": 2.92, + "theoretical_loss": 3.736424224378, + "tokens_seen": 784465920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003850050150451354, + "loss": 2.9541, + "theoretical_loss": 3.736393172732349, + "tokens_seen": 784531456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849949849548646, + "loss": 2.8379, + "theoretical_loss": 3.7363621244067193, + "tokens_seen": 784596992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849849548645938, + "loss": 2.7373, + "theoretical_loss": 3.736331079400479, + "tokens_seen": 784662528 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384974924774323, + "loss": 2.803, + "theoretical_loss": 3.7363000377129962, + "tokens_seen": 784728064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 440843, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.821906566619873, + "objective/train/theoretical_loss": 3.7362689993436398, + "objective/train/tokens_used": 805253600, + "theoretical_loss": 3.7362689993436398, + "tokens_seen": 784793600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038496489468405217, + "loss": 2.7818, + "theoretical_loss": 3.7362689993436398, + "tokens_seen": 784793600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038495486459378135, + "loss": 2.7273, + "theoretical_loss": 3.7362379642917767, + "tokens_seen": 784859136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038494483450351053, + "loss": 2.6269, + "theoretical_loss": 3.736206932556776, + "tokens_seen": 784924672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038493480441323977, + "loss": 2.8092, + "theoretical_loss": 3.7361759041380065, + "tokens_seen": 784990208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003849247743229689, + "loss": 2.8882, + "theoretical_loss": 3.736144879034837, + "tokens_seen": 785055744 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038491474423269813, + "loss": 2.5399, + "theoretical_loss": 3.736113857246636, + "tokens_seen": 785121280 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038490471414242725, + "loss": 2.6503, + "theoretical_loss": 3.7360828387727727, + "tokens_seen": 785186816 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848946840521565, + "loss": 2.7638, + "theoretical_loss": 3.736051823612616, + "tokens_seen": 785252352 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038488465396188567, + "loss": 2.6412, + "theoretical_loss": 3.736020811765537, + "tokens_seen": 785317888 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038487462387161485, + "loss": 2.8907, + "theoretical_loss": 3.735989803230904, + "tokens_seen": 785383424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038486459378134403, + "loss": 2.7288, + "theoretical_loss": 3.7359587980080873, + "tokens_seen": 785448960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038485456369107327, + "loss": 2.9444, + "theoretical_loss": 3.7359277960964565, + "tokens_seen": 785514496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848445336008024, + "loss": 2.9635, + "theoretical_loss": 3.735896797495382, + "tokens_seen": 785580032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038483450351053163, + "loss": 2.8273, + "theoretical_loss": 3.735865802204234, + "tokens_seen": 785645568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038482447342026076, + "loss": 2.6847, + "theoretical_loss": 3.735834810222384, + "tokens_seen": 785711104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038481444332999, + "loss": 2.6525, + "theoretical_loss": 3.735803821549201, + "tokens_seen": 785776640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003848044132397192, + "loss": 2.6116, + "theoretical_loss": 3.735772836184057, + "tokens_seen": 785842176 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038479438314944836, + "loss": 2.8336, + "theoretical_loss": 3.7357418541263234, + "tokens_seen": 785907712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038478435305917754, + "loss": 2.9349, + "theoretical_loss": 3.7357108753753705, + "tokens_seen": 785973248 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847743229689067, + "loss": 2.7884, + "theoretical_loss": 3.7356798999305694, + "tokens_seen": 786038784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847642928786359, + "loss": 2.5847, + "theoretical_loss": 3.7356489277912925, + "tokens_seen": 786104320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038475426278836513, + "loss": 2.9119, + "theoretical_loss": 3.7356179589569116, + "tokens_seen": 786169856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038474423269809426, + "loss": 2.5322, + "theoretical_loss": 3.735586993426798, + "tokens_seen": 786235392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847342026078235, + "loss": 2.5874, + "theoretical_loss": 3.735556031200324, + "tokens_seen": 786300928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003847241725175527, + "loss": 2.9919, + "theoretical_loss": 3.7355250722768614, + "tokens_seen": 786366464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 442027, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.940617561340332, + "objective/train/theoretical_loss": 3.735494116655784, + "objective/train/tokens_used": 806892000, + "theoretical_loss": 3.735494116655784, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038471414242728186, + "loss": 2.9008, + "theoretical_loss": 3.735494116655784, + "tokens_seen": 786432000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038470411233701104, + "loss": 2.8244, + "theoretical_loss": 3.735463164336462, + "tokens_seen": 786497536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846940822467402, + "loss": 2.8714, + "theoretical_loss": 3.7354322153182706, + "tokens_seen": 786563072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846840521564694, + "loss": 2.9422, + "theoretical_loss": 3.7354012696005814, + "tokens_seen": 786628608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038467402206619864, + "loss": 2.7046, + "theoretical_loss": 3.735370327182768, + "tokens_seen": 786694144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038466399197592776, + "loss": 2.5513, + "theoretical_loss": 3.7353393880642036, + "tokens_seen": 786759680 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384653961885657, + "loss": 2.9961, + "theoretical_loss": 3.7353084522442614, + "tokens_seen": 786825216 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846439317953861, + "loss": 2.7466, + "theoretical_loss": 3.735277519722315, + "tokens_seen": 786890752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038463390170511536, + "loss": 2.7936, + "theoretical_loss": 3.735246590497738, + "tokens_seen": 786956288 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038462387161484454, + "loss": 2.6091, + "theoretical_loss": 3.735215664569905, + "tokens_seen": 787021824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846138415245737, + "loss": 2.5691, + "theoretical_loss": 3.7351847419381894, + "tokens_seen": 787087360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003846038114343029, + "loss": 2.775, + "theoretical_loss": 3.7351538226019665, + "tokens_seen": 787152896 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845937813440321, + "loss": 2.9211, + "theoretical_loss": 3.7351229065606093, + "tokens_seen": 787218432 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038458375125376127, + "loss": 2.8669, + "theoretical_loss": 3.735091993813494, + "tokens_seen": 787283968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845737211634905, + "loss": 2.7544, + "theoretical_loss": 3.7350610843599945, + "tokens_seen": 787349504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038456369107321963, + "loss": 2.7132, + "theoretical_loss": 3.7350301781994855, + "tokens_seen": 787415040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038455366098294886, + "loss": 2.8449, + "theoretical_loss": 3.734999275331343, + "tokens_seen": 787480576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038454363089267805, + "loss": 2.8362, + "theoretical_loss": 3.7349683757549412, + "tokens_seen": 787546112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038453360080240723, + "loss": 2.607, + "theoretical_loss": 3.7349374794696573, + "tokens_seen": 787611648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845235707121364, + "loss": 2.7922, + "theoretical_loss": 3.734906586474865, + "tokens_seen": 787677184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003845135406218656, + "loss": 2.7689, + "theoretical_loss": 3.734875696769942, + "tokens_seen": 787742720 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038450351053159477, + "loss": 2.8311, + "theoretical_loss": 3.7348448103542626, + "tokens_seen": 787808256 + }, + { + "epoch": 0.24, + "learning_rate": 0.000384493480441324, + "loss": 2.7133, + "theoretical_loss": 3.734813927227204, + "tokens_seen": 787873792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038448345035105313, + "loss": 2.7541, + "theoretical_loss": 3.734783047388142, + "tokens_seen": 787939328 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038447342026078237, + "loss": 2.7583, + "theoretical_loss": 3.7347521708364537, + "tokens_seen": 788004864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 442587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.145890474319458, + "objective/train/theoretical_loss": 3.7347212975715154, + "objective/train/tokens_used": 808530400, + "theoretical_loss": 3.7347212975715154, + "tokens_seen": 788070400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844633901705115, + "loss": 2.9654, + "theoretical_loss": 3.7347212975715154, + "tokens_seen": 788070400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038445336008024073, + "loss": 2.7115, + "theoretical_loss": 3.734690427592704, + "tokens_seen": 788135936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844433299899699, + "loss": 2.3644, + "theoretical_loss": 3.7346595608993964, + "tokens_seen": 788201472 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844332998996991, + "loss": 2.6955, + "theoretical_loss": 3.73462869749097, + "tokens_seen": 788267008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003844232698094283, + "loss": 2.8928, + "theoretical_loss": 3.7345978373668025, + "tokens_seen": 788332544 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038441323971915745, + "loss": 2.5706, + "theoretical_loss": 3.73456698052627, + "tokens_seen": 788398080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038440320962888664, + "loss": 2.6816, + "theoretical_loss": 3.7345361269687523, + "tokens_seen": 788463616 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038439317953861587, + "loss": 2.6384, + "theoretical_loss": 3.7345052766936253, + "tokens_seen": 788529152 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038438314944834505, + "loss": 2.5129, + "theoretical_loss": 3.7344744297002688, + "tokens_seen": 788594688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038437311935807423, + "loss": 2.6681, + "theoretical_loss": 3.7344435859880596, + "tokens_seen": 788660224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038436308926780347, + "loss": 2.7354, + "theoretical_loss": 3.7344127455563765, + "tokens_seen": 788725760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843530591775326, + "loss": 2.5316, + "theoretical_loss": 3.7343819084045977, + "tokens_seen": 788791296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038434302908726183, + "loss": 2.7936, + "theoretical_loss": 3.734351074532103, + "tokens_seen": 788856832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038433299899699096, + "loss": 2.7816, + "theoretical_loss": 3.7343202439382703, + "tokens_seen": 788922368 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843229689067202, + "loss": 2.5337, + "theoretical_loss": 3.734289416622479, + "tokens_seen": 788987904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003843129388164494, + "loss": 2.655, + "theoretical_loss": 3.7342585925841085, + "tokens_seen": 789053440 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038430290872617856, + "loss": 2.6479, + "theoretical_loss": 3.734227771822538, + "tokens_seen": 789118976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038429287863590774, + "loss": 2.5965, + "theoretical_loss": 3.7341969543371474, + "tokens_seen": 789184512 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842828485456369, + "loss": 2.8551, + "theoretical_loss": 3.7341661401273156, + "tokens_seen": 789250048 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842728184553661, + "loss": 2.9743, + "theoretical_loss": 3.734135329192423, + "tokens_seen": 789315584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038426278836509533, + "loss": 2.6725, + "theoretical_loss": 3.73410452153185, + "tokens_seen": 789381120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038425275827482446, + "loss": 2.8066, + "theoretical_loss": 3.734073717144976, + "tokens_seen": 789446656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842427281845537, + "loss": 2.9044, + "theoretical_loss": 3.734042916031182, + "tokens_seen": 789512192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842326980942829, + "loss": 2.7009, + "theoretical_loss": 3.734012118189849, + "tokens_seen": 789577728 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038422266800401206, + "loss": 2.6561, + "theoretical_loss": 3.7339813236203567, + "tokens_seen": 789643264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 443777, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3760197162628174, + "objective/train/theoretical_loss": 3.733950532322087, + "objective/train/tokens_used": 810168800, + "theoretical_loss": 3.733950532322087, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038421263791374124, + "loss": 3.0686, + "theoretical_loss": 3.733950532322087, + "tokens_seen": 789708800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003842026078234704, + "loss": 2.6925, + "theoretical_loss": 3.73391974429442, + "tokens_seen": 789774336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841925777331996, + "loss": 2.6252, + "theoretical_loss": 3.7338889595367375, + "tokens_seen": 789839872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038418254764292884, + "loss": 2.6106, + "theoretical_loss": 3.733858178048421, + "tokens_seen": 789905408 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038417251755265796, + "loss": 2.6877, + "theoretical_loss": 3.7338273998288516, + "tokens_seen": 789970944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841624874623872, + "loss": 2.6514, + "theoretical_loss": 3.733796624877412, + "tokens_seen": 790036480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841524573721163, + "loss": 2.515, + "theoretical_loss": 3.7337658531934834, + "tokens_seen": 790102016 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038414242728184556, + "loss": 2.8697, + "theoretical_loss": 3.733735084776448, + "tokens_seen": 790167552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038413239719157474, + "loss": 2.6622, + "theoretical_loss": 3.733704319625688, + "tokens_seen": 790233088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841223671013039, + "loss": 2.8917, + "theoretical_loss": 3.7336735577405857, + "tokens_seen": 790298624 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841123370110331, + "loss": 2.5297, + "theoretical_loss": 3.733642799120524, + "tokens_seen": 790364160 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003841023069207623, + "loss": 2.7386, + "theoretical_loss": 3.733612043764886, + "tokens_seen": 790429696 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038409227683049147, + "loss": 2.8363, + "theoretical_loss": 3.7335812916730537, + "tokens_seen": 790495232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840822467402207, + "loss": 2.917, + "theoretical_loss": 3.733550542844411, + "tokens_seen": 790560768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038407221664994983, + "loss": 2.7862, + "theoretical_loss": 3.7335197972783405, + "tokens_seen": 790626304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038406218655967907, + "loss": 2.6633, + "theoretical_loss": 3.7334890549742257, + "tokens_seen": 790691840 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038405215646940825, + "loss": 2.7952, + "theoretical_loss": 3.7334583159314514, + "tokens_seen": 790757376 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038404212637913743, + "loss": 2.7748, + "theoretical_loss": 3.7334275801494, + "tokens_seen": 790822912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840320962888666, + "loss": 2.6345, + "theoretical_loss": 3.733396847627456, + "tokens_seen": 790888448 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840220661985958, + "loss": 2.6611, + "theoretical_loss": 3.7333661183650033, + "tokens_seen": 790953984 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038401203610832497, + "loss": 2.7081, + "theoretical_loss": 3.7333353923614263, + "tokens_seen": 791019520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003840020060180542, + "loss": 2.5762, + "theoretical_loss": 3.7333046696161087, + "tokens_seen": 791085056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038399197592778333, + "loss": 2.9757, + "theoretical_loss": 3.733273950128437, + "tokens_seen": 791150592 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038398194583751257, + "loss": 2.7254, + "theoretical_loss": 3.733243233897794, + "tokens_seen": 791216128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839719157472417, + "loss": 2.7849, + "theoretical_loss": 3.7332125209235656, + "tokens_seen": 791281664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 444379, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.697193145751953, + "objective/train/theoretical_loss": 3.7331818112051365, + "objective/train/tokens_used": 811807200, + "theoretical_loss": 3.7331818112051365, + "tokens_seen": 791347200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038396188565697093, + "loss": 2.9443, + "theoretical_loss": 3.7331818112051365, + "tokens_seen": 791347200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839518555667001, + "loss": 2.7015, + "theoretical_loss": 3.7331511047418924, + "tokens_seen": 791412736 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839418254764293, + "loss": 2.8082, + "theoretical_loss": 3.7331204015332187, + "tokens_seen": 791478272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003839317953861585, + "loss": 2.6834, + "theoretical_loss": 3.7330897015785003, + "tokens_seen": 791543808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038392176529588765, + "loss": 2.7401, + "theoretical_loss": 3.733059004877124, + "tokens_seen": 791609344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038391173520561684, + "loss": 2.4514, + "theoretical_loss": 3.733028311428475, + "tokens_seen": 791674880 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038390170511534607, + "loss": 2.6914, + "theoretical_loss": 3.73299762123194, + "tokens_seen": 791740416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838916750250752, + "loss": 2.843, + "theoretical_loss": 3.732966934286904, + "tokens_seen": 791805952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038388164493480443, + "loss": 2.693, + "theoretical_loss": 3.7329362505927555, + "tokens_seen": 791871488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838716148445336, + "loss": 2.7452, + "theoretical_loss": 3.73290557014888, + "tokens_seen": 791937024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838615847542628, + "loss": 2.4392, + "theoretical_loss": 3.732874892954663, + "tokens_seen": 792002560 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383851554663992, + "loss": 2.6381, + "theoretical_loss": 3.7328442190094937, + "tokens_seen": 792068096 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038384152457372116, + "loss": 2.7508, + "theoretical_loss": 3.732813548312758, + "tokens_seen": 792133632 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038383149448345034, + "loss": 2.6207, + "theoretical_loss": 3.7327828808638435, + "tokens_seen": 792199168 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838214643931796, + "loss": 2.7213, + "theoretical_loss": 3.732752216662137, + "tokens_seen": 792264704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003838114343029087, + "loss": 2.6121, + "theoretical_loss": 3.732721555707027, + "tokens_seen": 792330240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038380140421263794, + "loss": 2.937, + "theoretical_loss": 3.7326908979979008, + "tokens_seen": 792395776 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038379137412236706, + "loss": 2.624, + "theoretical_loss": 3.7326602435341467, + "tokens_seen": 792461312 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837813440320963, + "loss": 2.928, + "theoretical_loss": 3.732629592315152, + "tokens_seen": 792526848 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837713139418255, + "loss": 2.8103, + "theoretical_loss": 3.732598944340306, + "tokens_seen": 792592384 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038376128385155466, + "loss": 2.6622, + "theoretical_loss": 3.732568299608996, + "tokens_seen": 792657920 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038375125376128384, + "loss": 2.7235, + "theoretical_loss": 3.732537658120612, + "tokens_seen": 792723456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837412236710131, + "loss": 2.7727, + "theoretical_loss": 3.7325070198745416, + "tokens_seen": 792788992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837311935807422, + "loss": 2.8658, + "theoretical_loss": 3.732476384870174, + "tokens_seen": 792854528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038372116349047144, + "loss": 2.6207, + "theoretical_loss": 3.732445753106899, + "tokens_seen": 792920064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 445600, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.453542947769165, + "objective/train/theoretical_loss": 3.7324151245841044, + "objective/train/tokens_used": 813445600, + "theoretical_loss": 3.7324151245841044, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038371113340020057, + "loss": 2.8558, + "theoretical_loss": 3.7324151245841044, + "tokens_seen": 792985600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003837011033099298, + "loss": 2.7573, + "theoretical_loss": 3.7323844993011814, + "tokens_seen": 793051136 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383691073219659, + "loss": 2.7434, + "theoretical_loss": 3.732353877257518, + "tokens_seen": 793116672 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038368104312938816, + "loss": 2.7893, + "theoretical_loss": 3.732323258452505, + "tokens_seen": 793182208 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038367101303911735, + "loss": 2.8657, + "theoretical_loss": 3.7322926428855316, + "tokens_seen": 793247744 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836609829488465, + "loss": 2.8876, + "theoretical_loss": 3.7322620305559893, + "tokens_seen": 793313280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836509528585757, + "loss": 2.7263, + "theoretical_loss": 3.732231421463266, + "tokens_seen": 793378816 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038364092276830494, + "loss": 2.7745, + "theoretical_loss": 3.7322008156067543, + "tokens_seen": 793444352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836308926780341, + "loss": 2.8643, + "theoretical_loss": 3.732170212985844, + "tokens_seen": 793509888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836208625877633, + "loss": 2.6559, + "theoretical_loss": 3.7321396135999256, + "tokens_seen": 793575424 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003836108324974925, + "loss": 2.6391, + "theoretical_loss": 3.73210901744839, + "tokens_seen": 793640960 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038360080240722167, + "loss": 2.76, + "theoretical_loss": 3.732078424530629, + "tokens_seen": 793706496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835907723169509, + "loss": 2.8954, + "theoretical_loss": 3.732047834846033, + "tokens_seen": 793772032 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038358074222668003, + "loss": 2.6983, + "theoretical_loss": 3.732017248393994, + "tokens_seen": 793837568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038357071213640927, + "loss": 2.628, + "theoretical_loss": 3.7319866651739035, + "tokens_seen": 793903104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038356068204613845, + "loss": 2.7902, + "theoretical_loss": 3.7319560851851525, + "tokens_seen": 793968640 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038355065195586763, + "loss": 2.6634, + "theoretical_loss": 3.7319255084271337, + "tokens_seen": 794034176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835406218655968, + "loss": 2.8909, + "theoretical_loss": 3.731894934899239, + "tokens_seen": 794099712 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383530591775326, + "loss": 2.7643, + "theoretical_loss": 3.7318643646008605, + "tokens_seen": 794165248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038352056168505517, + "loss": 2.7051, + "theoretical_loss": 3.731833797531391, + "tokens_seen": 794230784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003835105315947844, + "loss": 2.8871, + "theoretical_loss": 3.731803233690222, + "tokens_seen": 794296320 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038350050150451353, + "loss": 2.9249, + "theoretical_loss": 3.7317726730767475, + "tokens_seen": 794361856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038349047141424277, + "loss": 2.7298, + "theoretical_loss": 3.73174211569036, + "tokens_seen": 794427392 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834804413239719, + "loss": 2.8595, + "theoretical_loss": 3.731711561530453, + "tokens_seen": 794492928 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038347041123370113, + "loss": 2.9234, + "theoretical_loss": 3.731681010596418, + "tokens_seen": 794558464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 446700, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2344651222229004, + "objective/train/theoretical_loss": 3.73165046288765, + "objective/train/tokens_used": 815084000, + "theoretical_loss": 3.73165046288765, + "tokens_seen": 794624000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834603811434303, + "loss": 2.5192, + "theoretical_loss": 3.73165046288765, + "tokens_seen": 794624000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834503510531595, + "loss": 2.7426, + "theoretical_loss": 3.731619918403542, + "tokens_seen": 794689536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834403209628887, + "loss": 2.6839, + "theoretical_loss": 3.731589377143488, + "tokens_seen": 794755072 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038343029087261786, + "loss": 2.638, + "theoretical_loss": 3.7315588391068815, + "tokens_seen": 794820608 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038342026078234704, + "loss": 2.7918, + "theoretical_loss": 3.731528304293117, + "tokens_seen": 794886144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038341023069207627, + "loss": 2.7242, + "theoretical_loss": 3.731497772701588, + "tokens_seen": 794951680 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003834002006018054, + "loss": 2.7139, + "theoretical_loss": 3.7314672443316894, + "tokens_seen": 795017216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038339017051153463, + "loss": 2.8092, + "theoretical_loss": 3.7314367191828155, + "tokens_seen": 795082752 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833801404212638, + "loss": 2.3607, + "theoretical_loss": 3.731406197254361, + "tokens_seen": 795148288 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383370110330993, + "loss": 2.7424, + "theoretical_loss": 3.7313756785457217, + "tokens_seen": 795213824 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833600802407222, + "loss": 2.669, + "theoretical_loss": 3.7313451630562913, + "tokens_seen": 795279360 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038335005015045136, + "loss": 2.8487, + "theoretical_loss": 3.731314650785465, + "tokens_seen": 795344896 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038334002006018054, + "loss": 2.7453, + "theoretical_loss": 3.7312841417326394, + "tokens_seen": 795410432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833299899699098, + "loss": 2.7604, + "theoretical_loss": 3.7312536358972084, + "tokens_seen": 795475968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003833199598796389, + "loss": 2.5242, + "theoretical_loss": 3.7312231332785695, + "tokens_seen": 795541504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038330992978936814, + "loss": 2.9914, + "theoretical_loss": 3.731192633876117, + "tokens_seen": 795607040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038329989969909726, + "loss": 2.7639, + "theoretical_loss": 3.7311621376892474, + "tokens_seen": 795672576 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832898696088265, + "loss": 2.9645, + "theoretical_loss": 3.7311316447173573, + "tokens_seen": 795738112 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832798395185557, + "loss": 2.8367, + "theoretical_loss": 3.731101154959842, + "tokens_seen": 795803648 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038326980942828486, + "loss": 2.8634, + "theoretical_loss": 3.7310706684160992, + "tokens_seen": 795869184 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038325977933801404, + "loss": 2.6604, + "theoretical_loss": 3.731040185085525, + "tokens_seen": 795934720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832497492477433, + "loss": 2.6438, + "theoretical_loss": 3.731009704967516, + "tokens_seen": 796000256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003832397191574724, + "loss": 2.8332, + "theoretical_loss": 3.7309792280614698, + "tokens_seen": 796065792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038322968906720164, + "loss": 2.6339, + "theoretical_loss": 3.730948754366783, + "tokens_seen": 796131328 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038321965897693077, + "loss": 2.6891, + "theoretical_loss": 3.730918283882853, + "tokens_seen": 796196864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 447326, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5912489891052246, + "objective/train/theoretical_loss": 3.730887816609077, + "objective/train/tokens_used": 816722400, + "theoretical_loss": 3.730887816609077, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038320962888666, + "loss": 2.8906, + "theoretical_loss": 3.730887816609077, + "tokens_seen": 796262400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831995987963892, + "loss": 2.6539, + "theoretical_loss": 3.730857352544853, + "tokens_seen": 796327936 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038318956870611836, + "loss": 2.6463, + "theoretical_loss": 3.7308268916895795, + "tokens_seen": 796393472 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038317953861584755, + "loss": 3.0121, + "theoretical_loss": 3.730796434042653, + "tokens_seen": 796459008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831695085255767, + "loss": 2.5218, + "theoretical_loss": 3.730765979603473, + "tokens_seen": 796524544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831594784353059, + "loss": 2.6623, + "theoretical_loss": 3.7307355283714365, + "tokens_seen": 796590080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038314944834503514, + "loss": 2.9648, + "theoretical_loss": 3.7307050803459427, + "tokens_seen": 796655616 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038313941825476427, + "loss": 2.9332, + "theoretical_loss": 3.73067463552639, + "tokens_seen": 796721152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003831293881644935, + "loss": 2.6942, + "theoretical_loss": 3.7306441939121773, + "tokens_seen": 796786688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038311935807422263, + "loss": 2.7156, + "theoretical_loss": 3.7306137555027035, + "tokens_seen": 796852224 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038310932798395187, + "loss": 2.6772, + "theoretical_loss": 3.7305833202973675, + "tokens_seen": 796917760 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038309929789368105, + "loss": 2.8115, + "theoretical_loss": 3.730552888295569, + "tokens_seen": 796983296 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038308926780341023, + "loss": 2.8397, + "theoretical_loss": 3.730522459496707, + "tokens_seen": 797048832 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830792377131394, + "loss": 2.7367, + "theoretical_loss": 3.7304920339001812, + "tokens_seen": 797114368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038306920762286865, + "loss": 2.7863, + "theoretical_loss": 3.7304616115053917, + "tokens_seen": 797179904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003830591775325978, + "loss": 2.8075, + "theoretical_loss": 3.7304311923117375, + "tokens_seen": 797245440 + }, + { + "epoch": 0.24, + "learning_rate": 0.000383049147442327, + "loss": 2.631, + "theoretical_loss": 3.7304007763186195, + "tokens_seen": 797310976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038303911735205614, + "loss": 2.5869, + "theoretical_loss": 3.730370363525438, + "tokens_seen": 797376512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038302908726178537, + "loss": 2.8324, + "theoretical_loss": 3.7303399539315927, + "tokens_seen": 797442048 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038301905717151455, + "loss": 2.586, + "theoretical_loss": 3.7303095475364842, + "tokens_seen": 797507584 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038300902708124373, + "loss": 2.7626, + "theoretical_loss": 3.730279144339514, + "tokens_seen": 797573120 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829989969909729, + "loss": 2.8478, + "theoretical_loss": 3.7302487443400825, + "tokens_seen": 797638656 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829889669007021, + "loss": 2.798, + "theoretical_loss": 3.7302183475375905, + "tokens_seen": 797704192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829789368104313, + "loss": 2.6434, + "theoretical_loss": 3.7301879539314395, + "tokens_seen": 797769728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829689067201605, + "loss": 2.6643, + "theoretical_loss": 3.730157563521031, + "tokens_seen": 797835264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 447858, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1472742557525635, + "objective/train/theoretical_loss": 3.730127176305765, + "objective/train/tokens_used": 818360800, + "theoretical_loss": 3.730127176305765, + "tokens_seen": 797900800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038295887662988964, + "loss": 2.845, + "theoretical_loss": 3.730127176305765, + "tokens_seen": 797900800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829488465396189, + "loss": 2.7808, + "theoretical_loss": 3.730096792285046, + "tokens_seen": 797966336 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382938816449348, + "loss": 2.6188, + "theoretical_loss": 3.7300664114582736, + "tokens_seen": 798031872 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038292878635907724, + "loss": 2.8079, + "theoretical_loss": 3.7300360338248506, + "tokens_seen": 798097408 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829187562688064, + "loss": 2.9146, + "theoretical_loss": 3.7300056593841795, + "tokens_seen": 798162944 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003829087261785356, + "loss": 2.9053, + "theoretical_loss": 3.7299752881356616, + "tokens_seen": 798228480 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828986960882648, + "loss": 2.5042, + "theoretical_loss": 3.7299449200787, + "tokens_seen": 798294016 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382888665997994, + "loss": 2.6706, + "theoretical_loss": 3.729914555212698, + "tokens_seen": 798359552 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828786359077232, + "loss": 2.3704, + "theoretical_loss": 3.7298841935370572, + "tokens_seen": 798425088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828686058174524, + "loss": 2.6598, + "theoretical_loss": 3.729853835051181, + "tokens_seen": 798490624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038285857572718156, + "loss": 2.8312, + "theoretical_loss": 3.729823479754473, + "tokens_seen": 798556160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038284854563691074, + "loss": 2.8285, + "theoretical_loss": 3.729793127646336, + "tokens_seen": 798621696 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038283851554664, + "loss": 2.6438, + "theoretical_loss": 3.7297627787261733, + "tokens_seen": 798687232 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003828284854563691, + "loss": 2.6833, + "theoretical_loss": 3.729732432993389, + "tokens_seen": 798752768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038281845536609834, + "loss": 2.7391, + "theoretical_loss": 3.729702090447387, + "tokens_seen": 798818304 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038280842527582746, + "loss": 2.4969, + "theoretical_loss": 3.7296717510875705, + "tokens_seen": 798883840 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827983951855567, + "loss": 2.7266, + "theoretical_loss": 3.7296414149133437, + "tokens_seen": 798949376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827883650952859, + "loss": 2.7095, + "theoretical_loss": 3.7296110819241117, + "tokens_seen": 799014912 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038277833500501506, + "loss": 2.6529, + "theoretical_loss": 3.729580752119278, + "tokens_seen": 799080448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038276830491474424, + "loss": 2.774, + "theoretical_loss": 3.729550425498248, + "tokens_seen": 799145984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827582748244735, + "loss": 2.671, + "theoretical_loss": 3.729520102060425, + "tokens_seen": 799211520 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827482447342026, + "loss": 2.6735, + "theoretical_loss": 3.7294897818052157, + "tokens_seen": 799277056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038273821464393184, + "loss": 2.6403, + "theoretical_loss": 3.729459464732024, + "tokens_seen": 799342592 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038272818455366097, + "loss": 2.7248, + "theoretical_loss": 3.7294291508402555, + "tokens_seen": 799408128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827181544633902, + "loss": 2.765, + "theoretical_loss": 3.729398840129315, + "tokens_seen": 799473664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 448758, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5953433513641357, + "objective/train/theoretical_loss": 3.729368532598609, + "objective/train/tokens_used": 819999200, + "theoretical_loss": 3.729368532598609, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003827081243731194, + "loss": 2.7885, + "theoretical_loss": 3.729368532598609, + "tokens_seen": 799539200 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038269809428284856, + "loss": 2.8063, + "theoretical_loss": 3.729338228247543, + "tokens_seen": 799604736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038268806419257775, + "loss": 2.7712, + "theoretical_loss": 3.7293079270755216, + "tokens_seen": 799670272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826780341023069, + "loss": 2.814, + "theoretical_loss": 3.7292776290819525, + "tokens_seen": 799735808 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826680040120361, + "loss": 2.8649, + "theoretical_loss": 3.729247334266241, + "tokens_seen": 799801344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038265797392176534, + "loss": 2.8075, + "theoretical_loss": 3.7292170426277935, + "tokens_seen": 799866880 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038264794383149447, + "loss": 2.6499, + "theoretical_loss": 3.7291867541660166, + "tokens_seen": 799932416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003826379137412237, + "loss": 2.7997, + "theoretical_loss": 3.729156468880317, + "tokens_seen": 799997952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038262788365095283, + "loss": 2.6938, + "theoretical_loss": 3.729126186770101, + "tokens_seen": 800063488 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038261785356068207, + "loss": 2.9072, + "theoretical_loss": 3.7290959078347763, + "tokens_seen": 800129024 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038260782347041125, + "loss": 2.6993, + "theoretical_loss": 3.72906563207375, + "tokens_seen": 800194560 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038259779338014043, + "loss": 2.5006, + "theoretical_loss": 3.7290353594864287, + "tokens_seen": 800260096 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825877632898696, + "loss": 3.0636, + "theoretical_loss": 3.7290050900722203, + "tokens_seen": 800325632 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038257773319959885, + "loss": 2.8181, + "theoretical_loss": 3.7289748238305327, + "tokens_seen": 800391168 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382567703109328, + "loss": 2.6539, + "theoretical_loss": 3.7289445607607727, + "tokens_seen": 800456704 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825576730190572, + "loss": 2.9976, + "theoretical_loss": 3.728914300862349, + "tokens_seen": 800522240 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038254764292878634, + "loss": 2.7945, + "theoretical_loss": 3.72888404413467, + "tokens_seen": 800587776 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038253761283851557, + "loss": 2.7775, + "theoretical_loss": 3.7288537905771433, + "tokens_seen": 800653312 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038252758274824475, + "loss": 2.5832, + "theoretical_loss": 3.728823540189177, + "tokens_seen": 800718848 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038251755265797393, + "loss": 2.8424, + "theoretical_loss": 3.728793292970181, + "tokens_seen": 800784384 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003825075225677031, + "loss": 3.0617, + "theoretical_loss": 3.7287630489195625, + "tokens_seen": 800849920 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824974924774323, + "loss": 2.771, + "theoretical_loss": 3.728732808036731, + "tokens_seen": 800915456 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824874623871615, + "loss": 2.8826, + "theoretical_loss": 3.7287025703210954, + "tokens_seen": 800980992 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824774322968907, + "loss": 2.9558, + "theoretical_loss": 3.7286723357720657, + "tokens_seen": 801046528 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038246740220661984, + "loss": 2.7418, + "theoretical_loss": 3.72864210438905, + "tokens_seen": 801112064 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 449233, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.920238733291626, + "objective/train/theoretical_loss": 3.728611876171459, + "objective/train/tokens_used": 821637600, + "theoretical_loss": 3.728611876171459, + "tokens_seen": 801177600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824573721163491, + "loss": 2.7952, + "theoretical_loss": 3.728611876171459, + "tokens_seen": 801177600 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824473420260782, + "loss": 2.6668, + "theoretical_loss": 3.7285816511187013, + "tokens_seen": 801243136 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038243731193580744, + "loss": 2.7591, + "theoretical_loss": 3.7285514292301873, + "tokens_seen": 801308672 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824272818455366, + "loss": 2.7631, + "theoretical_loss": 3.728521210505327, + "tokens_seen": 801374208 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003824172517552658, + "loss": 2.7571, + "theoretical_loss": 3.7284909949435305, + "tokens_seen": 801439744 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382407221664995, + "loss": 2.655, + "theoretical_loss": 3.728460782544208, + "tokens_seen": 801505280 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823971915747242, + "loss": 2.884, + "theoretical_loss": 3.7284305733067704, + "tokens_seen": 801570816 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038238716148445334, + "loss": 2.7354, + "theoretical_loss": 3.728400367230627, + "tokens_seen": 801636352 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823771313941826, + "loss": 2.7962, + "theoretical_loss": 3.7283701643151903, + "tokens_seen": 801701888 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823671013039117, + "loss": 2.7577, + "theoretical_loss": 3.7283399645598703, + "tokens_seen": 801767424 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038235707121364094, + "loss": 2.929, + "theoretical_loss": 3.728309767964078, + "tokens_seen": 801832960 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823470411233701, + "loss": 2.6252, + "theoretical_loss": 3.728279574527225, + "tokens_seen": 801898496 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823370110330993, + "loss": 2.6991, + "theoretical_loss": 3.7282493842487225, + "tokens_seen": 801964032 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003823269809428285, + "loss": 2.6825, + "theoretical_loss": 3.7282191971279826, + "tokens_seen": 802029568 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038231695085255766, + "loss": 2.6514, + "theoretical_loss": 3.7281890131644158, + "tokens_seen": 802095104 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038230692076228685, + "loss": 2.7606, + "theoretical_loss": 3.728158832357435, + "tokens_seen": 802160640 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822968906720161, + "loss": 2.7181, + "theoretical_loss": 3.7281286547064525, + "tokens_seen": 802226176 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822868605817452, + "loss": 2.7173, + "theoretical_loss": 3.7280984802108796, + "tokens_seen": 802291712 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038227683049147444, + "loss": 2.7012, + "theoretical_loss": 3.7280683088701294, + "tokens_seen": 802357248 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038226680040120357, + "loss": 2.6665, + "theoretical_loss": 3.728038140683614, + "tokens_seen": 802422784 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822567703109328, + "loss": 2.7573, + "theoretical_loss": 3.728007975650746, + "tokens_seen": 802488320 + }, + { + "epoch": 0.24, + "learning_rate": 0.000382246740220662, + "loss": 2.8618, + "theoretical_loss": 3.727977813770938, + "tokens_seen": 802553856 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038223671013039117, + "loss": 2.6473, + "theoretical_loss": 3.7279476550436037, + "tokens_seen": 802619392 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038222668004012035, + "loss": 2.6769, + "theoretical_loss": 3.727917499468156, + "tokens_seen": 802684928 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822166499498496, + "loss": 2.8372, + "theoretical_loss": 3.7278873470440077, + "tokens_seen": 802750464 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 450645, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0350584983825684, + "objective/train/theoretical_loss": 3.7278571977705734, + "objective/train/tokens_used": 823276000, + "theoretical_loss": 3.7278571977705734, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003822066198595787, + "loss": 2.6396, + "theoretical_loss": 3.7278571977705734, + "tokens_seen": 802816000 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038219658976930795, + "loss": 2.7545, + "theoretical_loss": 3.727827051647265, + "tokens_seen": 802881536 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821865596790371, + "loss": 2.7913, + "theoretical_loss": 3.727796908673498, + "tokens_seen": 802947072 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821765295887663, + "loss": 2.7719, + "theoretical_loss": 3.727766768848685, + "tokens_seen": 803012608 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003821664994984955, + "loss": 2.6693, + "theoretical_loss": 3.727736632172241, + "tokens_seen": 803078144 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038215646940822467, + "loss": 2.6396, + "theoretical_loss": 3.7277064986435793, + "tokens_seen": 803143680 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038214643931795385, + "loss": 2.7354, + "theoretical_loss": 3.7276763682621157, + "tokens_seen": 803209216 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038213640922768303, + "loss": 2.6416, + "theoretical_loss": 3.7276462410272635, + "tokens_seen": 803274752 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038212637913741227, + "loss": 2.9243, + "theoretical_loss": 3.727616116938438, + "tokens_seen": 803340288 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038211634904714145, + "loss": 2.6483, + "theoretical_loss": 3.7275859959950535, + "tokens_seen": 803405824 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038210631895687063, + "loss": 2.7773, + "theoretical_loss": 3.727555878196526, + "tokens_seen": 803471360 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820962888665998, + "loss": 2.7851, + "theoretical_loss": 3.7275257635422694, + "tokens_seen": 803536896 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038208625877632905, + "loss": 2.5762, + "theoretical_loss": 3.7274956520317, + "tokens_seen": 803602432 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820762286860582, + "loss": 2.8452, + "theoretical_loss": 3.727465543664233, + "tokens_seen": 803667968 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820661985957874, + "loss": 2.7465, + "theoretical_loss": 3.7274354384392847, + "tokens_seen": 803733504 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038205616850551654, + "loss": 2.6667, + "theoretical_loss": 3.7274053363562696, + "tokens_seen": 803799040 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038204613841524577, + "loss": 2.8064, + "theoretical_loss": 3.7273752374146043, + "tokens_seen": 803864576 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038203610832497495, + "loss": 2.7668, + "theoretical_loss": 3.7273451416137053, + "tokens_seen": 803930112 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038202607823470413, + "loss": 2.7471, + "theoretical_loss": 3.7273150489529883, + "tokens_seen": 803995648 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820160481444333, + "loss": 2.863, + "theoretical_loss": 3.72728495943187, + "tokens_seen": 804061184 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003820060180541625, + "loss": 2.7216, + "theoretical_loss": 3.727254873049767, + "tokens_seen": 804126720 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819959879638917, + "loss": 2.6481, + "theoretical_loss": 3.7272247898060966, + "tokens_seen": 804192256 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819859578736209, + "loss": 2.6998, + "theoretical_loss": 3.7271947097002744, + "tokens_seen": 804257792 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038197592778335004, + "loss": 2.9825, + "theoretical_loss": 3.727164632731718, + "tokens_seen": 804323328 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819658976930793, + "loss": 3.0266, + "theoretical_loss": 3.727134558899845, + "tokens_seen": 804388864 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 451555, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6607892513275146, + "objective/train/theoretical_loss": 3.7271044882040725, + "objective/train/tokens_used": 824914400, + "theoretical_loss": 3.7271044882040725, + "tokens_seen": 804454400 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819558676028084, + "loss": 2.6691, + "theoretical_loss": 3.7271044882040725, + "tokens_seen": 804454400 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038194583751253764, + "loss": 2.9954, + "theoretical_loss": 3.7270744206438184, + "tokens_seen": 804519936 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819358074222668, + "loss": 3.0286, + "theoretical_loss": 3.7270443562184994, + "tokens_seen": 804585472 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381925777331996, + "loss": 2.8963, + "theoretical_loss": 3.7270142949275344, + "tokens_seen": 804651008 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819157472417252, + "loss": 2.7464, + "theoretical_loss": 3.726984236770341, + "tokens_seen": 804716544 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003819057171514544, + "loss": 2.6961, + "theoretical_loss": 3.7269541817463363, + "tokens_seen": 804782080 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038189568706118354, + "loss": 2.8959, + "theoretical_loss": 3.7269241298549405, + "tokens_seen": 804847616 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818856569709128, + "loss": 2.8578, + "theoretical_loss": 3.726894081095571, + "tokens_seen": 804913152 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818756268806419, + "loss": 2.7054, + "theoretical_loss": 3.7268640354676466, + "tokens_seen": 804978688 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038186559679037114, + "loss": 2.7952, + "theoretical_loss": 3.726833992970586, + "tokens_seen": 805044224 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818555667001003, + "loss": 2.8643, + "theoretical_loss": 3.726803953603808, + "tokens_seen": 805109760 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818455366098295, + "loss": 2.9446, + "theoretical_loss": 3.7267739173667316, + "tokens_seen": 805175296 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818355065195587, + "loss": 3.0333, + "theoretical_loss": 3.7267438842587763, + "tokens_seen": 805240832 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038182547642928786, + "loss": 2.7721, + "theoretical_loss": 3.726713854279361, + "tokens_seen": 805306368 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038181544633901705, + "loss": 2.7606, + "theoretical_loss": 3.7266838274279066, + "tokens_seen": 805371904 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003818054162487463, + "loss": 2.8632, + "theoretical_loss": 3.7266538037038313, + "tokens_seen": 805437440 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817953861584754, + "loss": 2.8549, + "theoretical_loss": 3.7266237831065556, + "tokens_seen": 805502976 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038178535606820464, + "loss": 2.9827, + "theoretical_loss": 3.7265937656354993, + "tokens_seen": 805568512 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038177532597793377, + "loss": 2.5919, + "theoretical_loss": 3.7265637512900827, + "tokens_seen": 805634048 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381765295887663, + "loss": 2.5312, + "theoretical_loss": 3.7265337400697263, + "tokens_seen": 805699584 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817552657973922, + "loss": 2.6245, + "theoretical_loss": 3.7265037319738497, + "tokens_seen": 805765120 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038174523570712137, + "loss": 2.6506, + "theoretical_loss": 3.7264737270018746, + "tokens_seen": 805830656 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038173520561685055, + "loss": 2.7696, + "theoretical_loss": 3.7264437251532216, + "tokens_seen": 805896192 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817251755265798, + "loss": 2.8109, + "theoretical_loss": 3.7264137264273107, + "tokens_seen": 805961728 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003817151454363089, + "loss": 2.7733, + "theoretical_loss": 3.726383730823564, + "tokens_seen": 806027264 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 452218, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7736008167266846, + "objective/train/theoretical_loss": 3.7263537383414023, + "objective/train/tokens_used": 826552800, + "theoretical_loss": 3.7263537383414023, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038170511534603815, + "loss": 2.6347, + "theoretical_loss": 3.7263537383414023, + "tokens_seen": 806092800 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816950852557673, + "loss": 2.6228, + "theoretical_loss": 3.7263237489802474, + "tokens_seen": 806158336 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816850551654965, + "loss": 2.7033, + "theoretical_loss": 3.72629376273952, + "tokens_seen": 806223872 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816750250752257, + "loss": 2.6791, + "theoretical_loss": 3.726263779618643, + "tokens_seen": 806289408 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038166499498495487, + "loss": 2.7004, + "theoretical_loss": 3.726233799617037, + "tokens_seen": 806354944 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038165496489468405, + "loss": 2.6775, + "theoretical_loss": 3.7262038227341248, + "tokens_seen": 806420480 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038164493480441323, + "loss": 2.7478, + "theoretical_loss": 3.726173848969329, + "tokens_seen": 806486016 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816349047141424, + "loss": 2.7054, + "theoretical_loss": 3.72614387832207, + "tokens_seen": 806551552 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038162487462387165, + "loss": 2.9783, + "theoretical_loss": 3.7261139107917725, + "tokens_seen": 806617088 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003816148445336008, + "loss": 2.7914, + "theoretical_loss": 3.726083946377858, + "tokens_seen": 806682624 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038160481444333, + "loss": 2.67, + "theoretical_loss": 3.726053985079749, + "tokens_seen": 806748160 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038159478435305914, + "loss": 2.6837, + "theoretical_loss": 3.7260240268968694, + "tokens_seen": 806813696 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815847542627884, + "loss": 2.6676, + "theoretical_loss": 3.7259940718286417, + "tokens_seen": 806879232 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038157472417251756, + "loss": 2.7729, + "theoretical_loss": 3.7259641198744893, + "tokens_seen": 806944768 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038156469408224674, + "loss": 2.7676, + "theoretical_loss": 3.7259341710338356, + "tokens_seen": 807010304 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815546639919759, + "loss": 2.7686, + "theoretical_loss": 3.7259042253061043, + "tokens_seen": 807075840 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038154463390170515, + "loss": 2.8106, + "theoretical_loss": 3.725874282690718, + "tokens_seen": 807141376 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815346038114343, + "loss": 2.7067, + "theoretical_loss": 3.7258443431871022, + "tokens_seen": 807206912 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815245737211635, + "loss": 2.9003, + "theoretical_loss": 3.7258144067946803, + "tokens_seen": 807272448 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038151454363089264, + "loss": 2.8686, + "theoretical_loss": 3.725784473512876, + "tokens_seen": 807337984 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003815045135406219, + "loss": 2.6977, + "theoretical_loss": 3.7257545433411137, + "tokens_seen": 807403520 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038149448345035106, + "loss": 2.5566, + "theoretical_loss": 3.7257246162788187, + "tokens_seen": 807469056 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038148445336008024, + "loss": 2.6849, + "theoretical_loss": 3.7256946923254146, + "tokens_seen": 807534592 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814744232698094, + "loss": 2.8655, + "theoretical_loss": 3.725664771480327, + "tokens_seen": 807600128 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814643931795386, + "loss": 2.8026, + "theoretical_loss": 3.7256348537429798, + "tokens_seen": 807665664 + }, + { + "epoch": 0.24, + "objective/train/docs_used": 453465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5026791095733643, + "objective/train/theoretical_loss": 3.7256049391127988, + "objective/train/tokens_used": 828191200, + "theoretical_loss": 3.7256049391127988, + "tokens_seen": 807731200 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814543630892678, + "loss": 2.7585, + "theoretical_loss": 3.7256049391127988, + "tokens_seen": 807731200 + }, + { + "epoch": 0.24, + "learning_rate": 0.000381444332998997, + "loss": 2.5317, + "theoretical_loss": 3.7255750275892097, + "tokens_seen": 807796736 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038143430290872614, + "loss": 2.5894, + "theoretical_loss": 3.725545119171637, + "tokens_seen": 807862272 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003814242728184554, + "loss": 2.8322, + "theoretical_loss": 3.7255152138595067, + "tokens_seen": 807927808 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038141424272818456, + "loss": 2.89, + "theoretical_loss": 3.7254853116522444, + "tokens_seen": 807993344 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038140421263791374, + "loss": 2.7496, + "theoretical_loss": 3.7254554125492754, + "tokens_seen": 808058880 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813941825476429, + "loss": 2.8516, + "theoretical_loss": 3.7254255165500267, + "tokens_seen": 808124416 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813841524573721, + "loss": 2.6526, + "theoretical_loss": 3.725395623653924, + "tokens_seen": 808189952 + }, + { + "epoch": 0.24, + "learning_rate": 0.00038137412236710134, + "loss": 2.6249, + "theoretical_loss": 3.7253657338603934, + "tokens_seen": 808255488 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813640922768305, + "loss": 2.8297, + "theoretical_loss": 3.7253358471688616, + "tokens_seen": 808321024 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813540621865597, + "loss": 2.6693, + "theoretical_loss": 3.725305963578755, + "tokens_seen": 808386560 + }, + { + "epoch": 0.24, + "learning_rate": 0.0003813440320962889, + "loss": 2.7322, + "theoretical_loss": 3.725276083089501, + "tokens_seen": 808452096 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038133400200601806, + "loss": 2.7031, + "theoretical_loss": 3.725246205700526, + "tokens_seen": 808517632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038132397191574725, + "loss": 2.6764, + "theoretical_loss": 3.7252163314112567, + "tokens_seen": 808583168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003813139418254765, + "loss": 2.6776, + "theoretical_loss": 3.7251864602211207, + "tokens_seen": 808648704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003813039117352056, + "loss": 2.4349, + "theoretical_loss": 3.7251565921295455, + "tokens_seen": 808714240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038129388164493484, + "loss": 2.6752, + "theoretical_loss": 3.725126727135959, + "tokens_seen": 808779776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038128385155466397, + "loss": 2.7746, + "theoretical_loss": 3.7250968652397876, + "tokens_seen": 808845312 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812738214643932, + "loss": 2.8142, + "theoretical_loss": 3.7250670064404603, + "tokens_seen": 808910848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812637913741224, + "loss": 2.9367, + "theoretical_loss": 3.725037150737405, + "tokens_seen": 808976384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038125376128385157, + "loss": 2.7664, + "theoretical_loss": 3.725007298130049, + "tokens_seen": 809041920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038124373119358075, + "loss": 2.6014, + "theoretical_loss": 3.724977448617821, + "tokens_seen": 809107456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038123370110331, + "loss": 2.67, + "theoretical_loss": 3.72494760220015, + "tokens_seen": 809172992 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812236710130391, + "loss": 2.9132, + "theoretical_loss": 3.724917758876464, + "tokens_seen": 809238528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038121364092276835, + "loss": 2.8343, + "theoretical_loss": 3.724887918646192, + "tokens_seen": 809304064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 454216, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.918653726577759, + "objective/train/theoretical_loss": 3.7248580815087626, + "objective/train/tokens_used": 829829600, + "theoretical_loss": 3.7248580815087626, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003812036108324975, + "loss": 2.5661, + "theoretical_loss": 3.7248580815087626, + "tokens_seen": 809369600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811935807422267, + "loss": 2.5607, + "theoretical_loss": 3.724828247463605, + "tokens_seen": 809435136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811835506519559, + "loss": 2.6057, + "theoretical_loss": 3.7247984165101484, + "tokens_seen": 809500672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038117352056168507, + "loss": 2.5209, + "theoretical_loss": 3.724768588647822, + "tokens_seen": 809566208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038116349047141425, + "loss": 2.658, + "theoretical_loss": 3.724738763876056, + "tokens_seen": 809631744 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038115346038114343, + "loss": 2.7838, + "theoretical_loss": 3.724708942194279, + "tokens_seen": 809697280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811434302908726, + "loss": 2.648, + "theoretical_loss": 3.7246791236019217, + "tokens_seen": 809762816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038113340020060185, + "loss": 2.7635, + "theoretical_loss": 3.7246493080984133, + "tokens_seen": 809828352 + }, + { + "epoch": 0.25, + "learning_rate": 0.000381123370110331, + "loss": 2.8069, + "theoretical_loss": 3.724619495683185, + "tokens_seen": 809893888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003811133400200602, + "loss": 2.812, + "theoretical_loss": 3.724589686355665, + "tokens_seen": 809959424 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038110330992978934, + "loss": 2.706, + "theoretical_loss": 3.7245598801152857, + "tokens_seen": 810024960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810932798395186, + "loss": 2.8072, + "theoretical_loss": 3.7245300769614773, + "tokens_seen": 810090496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038108324974924776, + "loss": 2.7348, + "theoretical_loss": 3.7245002768936697, + "tokens_seen": 810156032 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038107321965897694, + "loss": 2.8901, + "theoretical_loss": 3.724470479911295, + "tokens_seen": 810221568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810631895687061, + "loss": 3.0223, + "theoretical_loss": 3.724440686013782, + "tokens_seen": 810287104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038105315947843535, + "loss": 2.6307, + "theoretical_loss": 3.724410895200565, + "tokens_seen": 810352640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810431293881645, + "loss": 2.8834, + "theoretical_loss": 3.724381107471072, + "tokens_seen": 810418176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810330992978937, + "loss": 2.4979, + "theoretical_loss": 3.724351322824737, + "tokens_seen": 810483712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038102306920762284, + "loss": 2.7276, + "theoretical_loss": 3.724321541260991, + "tokens_seen": 810549248 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003810130391173521, + "loss": 2.6296, + "theoretical_loss": 3.7242917627792647, + "tokens_seen": 810614784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038100300902708126, + "loss": 2.4256, + "theoretical_loss": 3.7242619873789913, + "tokens_seen": 810680320 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038099297893681044, + "loss": 2.7176, + "theoretical_loss": 3.724232215059603, + "tokens_seen": 810745856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809829488465396, + "loss": 2.7171, + "theoretical_loss": 3.72420244582053, + "tokens_seen": 810811392 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809729187562688, + "loss": 2.6742, + "theoretical_loss": 3.7241726796612067, + "tokens_seen": 810876928 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380962888665998, + "loss": 2.5437, + "theoretical_loss": 3.724142916581065, + "tokens_seen": 810942464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 455558, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9195406436920166, + "objective/train/theoretical_loss": 3.724113156579537, + "objective/train/tokens_used": 831468000, + "theoretical_loss": 3.724113156579537, + "tokens_seen": 811008000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809528585757272, + "loss": 2.731, + "theoretical_loss": 3.724113156579537, + "tokens_seen": 811008000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038094282848545635, + "loss": 2.6566, + "theoretical_loss": 3.7240833996560565, + "tokens_seen": 811073536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809327983951856, + "loss": 2.8637, + "theoretical_loss": 3.7240536458100557, + "tokens_seen": 811139072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038092276830491476, + "loss": 2.8568, + "theoretical_loss": 3.7240238950409683, + "tokens_seen": 811204608 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038091273821464394, + "loss": 2.5492, + "theoretical_loss": 3.723994147348227, + "tokens_seen": 811270144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003809027081243731, + "loss": 2.6746, + "theoretical_loss": 3.723964402731266, + "tokens_seen": 811335680 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808926780341023, + "loss": 2.7153, + "theoretical_loss": 3.723934661189518, + "tokens_seen": 811401216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808826479438315, + "loss": 2.8033, + "theoretical_loss": 3.7239049227224172, + "tokens_seen": 811466752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808726178535607, + "loss": 2.917, + "theoretical_loss": 3.723875187329398, + "tokens_seen": 811532288 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038086258776328985, + "loss": 2.8844, + "theoretical_loss": 3.7238454550098927, + "tokens_seen": 811597824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808525576730191, + "loss": 2.4595, + "theoretical_loss": 3.7238157257633366, + "tokens_seen": 811663360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808425275827482, + "loss": 2.9098, + "theoretical_loss": 3.7237859995891647, + "tokens_seen": 811728896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038083249749247745, + "loss": 2.7129, + "theoretical_loss": 3.72375627648681, + "tokens_seen": 811794432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808224674022066, + "loss": 2.668, + "theoretical_loss": 3.7237265564557083, + "tokens_seen": 811859968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003808124373119358, + "loss": 2.6803, + "theoretical_loss": 3.723696839495294, + "tokens_seen": 811925504 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380802407221665, + "loss": 2.7145, + "theoretical_loss": 3.723667125605002, + "tokens_seen": 811991040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038079237713139417, + "loss": 2.7764, + "theoretical_loss": 3.723637414784267, + "tokens_seen": 812056576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038078234704112335, + "loss": 2.6353, + "theoretical_loss": 3.723607707032525, + "tokens_seen": 812122112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807723169508526, + "loss": 2.8169, + "theoretical_loss": 3.7235780023492104, + "tokens_seen": 812187648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807622868605817, + "loss": 2.9368, + "theoretical_loss": 3.7235483007337593, + "tokens_seen": 812253184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038075225677031095, + "loss": 2.8544, + "theoretical_loss": 3.7235186021856075, + "tokens_seen": 812318720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038074222668004013, + "loss": 2.5776, + "theoretical_loss": 3.7234889067041905, + "tokens_seen": 812384256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807321965897693, + "loss": 2.8329, + "theoretical_loss": 3.723459214288944, + "tokens_seen": 812449792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807221664994985, + "loss": 2.7954, + "theoretical_loss": 3.723429524939305, + "tokens_seen": 812515328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003807121364092277, + "loss": 2.5979, + "theoretical_loss": 3.723399838654709, + "tokens_seen": 812580864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 456229, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.552311897277832, + "objective/train/theoretical_loss": 3.7233701554345924, + "objective/train/tokens_used": 833106400, + "theoretical_loss": 3.7233701554345924, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038070210631895685, + "loss": 2.6729, + "theoretical_loss": 3.7233701554345924, + "tokens_seen": 812646400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806920762286861, + "loss": 2.6202, + "theoretical_loss": 3.723340475278392, + "tokens_seen": 812711936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806820461384152, + "loss": 2.6352, + "theoretical_loss": 3.7233107981855453, + "tokens_seen": 812777472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038067201604814445, + "loss": 2.9071, + "theoretical_loss": 3.7232811241554877, + "tokens_seen": 812843008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806619859578736, + "loss": 2.8939, + "theoretical_loss": 3.7232514531876566, + "tokens_seen": 812908544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806519558676028, + "loss": 2.5899, + "theoretical_loss": 3.72322178528149, + "tokens_seen": 812974080 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380641925777332, + "loss": 2.7315, + "theoretical_loss": 3.7231921204364244, + "tokens_seen": 813039616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806318956870612, + "loss": 2.8286, + "theoretical_loss": 3.7231624586518977, + "tokens_seen": 813105152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806218655967904, + "loss": 2.5656, + "theoretical_loss": 3.723132799927347, + "tokens_seen": 813170688 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038061183550651954, + "loss": 2.7309, + "theoretical_loss": 3.7231031442622107, + "tokens_seen": 813236224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003806018054162488, + "loss": 2.7168, + "theoretical_loss": 3.723073491655926, + "tokens_seen": 813301760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038059177532597796, + "loss": 2.8353, + "theoretical_loss": 3.7230438421079315, + "tokens_seen": 813367296 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038058174523570714, + "loss": 2.542, + "theoretical_loss": 3.7230141956176652, + "tokens_seen": 813432832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805717151454363, + "loss": 2.7511, + "theoretical_loss": 3.722984552184565, + "tokens_seen": 813498368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038056168505516555, + "loss": 2.5704, + "theoretical_loss": 3.7229549118080705, + "tokens_seen": 813563904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805516549648947, + "loss": 2.5106, + "theoretical_loss": 3.7229252744876193, + "tokens_seen": 813629440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805416248746239, + "loss": 2.8, + "theoretical_loss": 3.7228956402226503, + "tokens_seen": 813694976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038053159478435304, + "loss": 2.5614, + "theoretical_loss": 3.722866009012603, + "tokens_seen": 813760512 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003805215646940823, + "loss": 2.7477, + "theoretical_loss": 3.7228363808569163, + "tokens_seen": 813826048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038051153460381146, + "loss": 2.7359, + "theoretical_loss": 3.722806755755029, + "tokens_seen": 813891584 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038050150451354064, + "loss": 2.8882, + "theoretical_loss": 3.722777133706381, + "tokens_seen": 813957120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804914744232698, + "loss": 2.7194, + "theoretical_loss": 3.7227475147104117, + "tokens_seen": 814022656 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380481444332999, + "loss": 2.6075, + "theoretical_loss": 3.7227178987665606, + "tokens_seen": 814088192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804714142427282, + "loss": 2.7222, + "theoretical_loss": 3.722688285874267, + "tokens_seen": 814153728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804613841524574, + "loss": 2.6867, + "theoretical_loss": 3.7226586760329727, + "tokens_seen": 814219264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 457261, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5911662578582764, + "objective/train/theoretical_loss": 3.7226290692421165, + "objective/train/tokens_used": 834744800, + "theoretical_loss": 3.7226290692421165, + "tokens_seen": 814284800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038045135406218655, + "loss": 2.6262, + "theoretical_loss": 3.7226290692421165, + "tokens_seen": 814284800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804413239719158, + "loss": 2.5049, + "theoretical_loss": 3.722599465501138, + "tokens_seen": 814350336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038043129388164496, + "loss": 2.7759, + "theoretical_loss": 3.722569864809479, + "tokens_seen": 814415872 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038042126379137414, + "loss": 2.6387, + "theoretical_loss": 3.722540267166579, + "tokens_seen": 814481408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804112337011033, + "loss": 2.712, + "theoretical_loss": 3.7225106725718797, + "tokens_seen": 814546944 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003804012036108325, + "loss": 2.7145, + "theoretical_loss": 3.722481081024821, + "tokens_seen": 814612480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803911735205617, + "loss": 2.8807, + "theoretical_loss": 3.7224514925248444, + "tokens_seen": 814678016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803811434302909, + "loss": 2.6498, + "theoretical_loss": 3.7224219070713915, + "tokens_seen": 814743552 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038037111334002005, + "loss": 2.8192, + "theoretical_loss": 3.722392324663903, + "tokens_seen": 814809088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803610832497493, + "loss": 2.6205, + "theoretical_loss": 3.7223627453018207, + "tokens_seen": 814874624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803510531594784, + "loss": 3.0639, + "theoretical_loss": 3.7223331689845853, + "tokens_seen": 814940160 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038034102306920765, + "loss": 2.5076, + "theoretical_loss": 3.7223035957116393, + "tokens_seen": 815005696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038033099297893683, + "loss": 2.4231, + "theoretical_loss": 3.722274025482425, + "tokens_seen": 815071232 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380320962888666, + "loss": 2.5898, + "theoretical_loss": 3.722244458296384, + "tokens_seen": 815136768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003803109327983952, + "loss": 2.756, + "theoretical_loss": 3.722214894152958, + "tokens_seen": 815202304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038030090270812437, + "loss": 2.8386, + "theoretical_loss": 3.72218533305159, + "tokens_seen": 815267840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038029087261785355, + "loss": 2.6173, + "theoretical_loss": 3.7221557749917222, + "tokens_seen": 815333376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802808425275828, + "loss": 2.808, + "theoretical_loss": 3.722126219972797, + "tokens_seen": 815398912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802708124373119, + "loss": 2.7507, + "theoretical_loss": 3.7220966679942578, + "tokens_seen": 815464448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038026078234704115, + "loss": 2.5634, + "theoretical_loss": 3.722067119055547, + "tokens_seen": 815529984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038025075225677033, + "loss": 2.7806, + "theoretical_loss": 3.7220375731561086, + "tokens_seen": 815595520 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802407221664995, + "loss": 2.6871, + "theoretical_loss": 3.722008030295384, + "tokens_seen": 815661056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802306920762287, + "loss": 2.5132, + "theoretical_loss": 3.7219784904728184, + "tokens_seen": 815726592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802206619859579, + "loss": 2.883, + "theoretical_loss": 3.7219489536878543, + "tokens_seen": 815792128 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038021063189568705, + "loss": 2.8185, + "theoretical_loss": 3.7219194199399355, + "tokens_seen": 815857664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 457884, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9456984996795654, + "objective/train/theoretical_loss": 3.721889889228506, + "objective/train/tokens_used": 836383200, + "theoretical_loss": 3.721889889228506, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003802006018054163, + "loss": 2.5694, + "theoretical_loss": 3.721889889228506, + "tokens_seen": 815923200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801905717151454, + "loss": 2.4299, + "theoretical_loss": 3.7218603615530093, + "tokens_seen": 815988736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038018054162487465, + "loss": 2.7284, + "theoretical_loss": 3.7218308369128903, + "tokens_seen": 816054272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801705115346038, + "loss": 2.5405, + "theoretical_loss": 3.7218013153075926, + "tokens_seen": 816119808 + }, + { + "epoch": 0.25, + "learning_rate": 0.000380160481444333, + "loss": 2.6405, + "theoretical_loss": 3.7217717967365607, + "tokens_seen": 816185344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801504513540622, + "loss": 2.5513, + "theoretical_loss": 3.7217422811992393, + "tokens_seen": 816250880 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801404212637914, + "loss": 2.761, + "theoretical_loss": 3.721712768695073, + "tokens_seen": 816316416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038013039117352056, + "loss": 2.5803, + "theoretical_loss": 3.7216832592235067, + "tokens_seen": 816381952 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038012036108324974, + "loss": 2.6744, + "theoretical_loss": 3.7216537527839852, + "tokens_seen": 816447488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003801103309929789, + "loss": 2.6138, + "theoretical_loss": 3.7216242493759535, + "tokens_seen": 816513024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038010030090270816, + "loss": 2.4472, + "theoretical_loss": 3.7215947489988572, + "tokens_seen": 816578560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800902708124373, + "loss": 2.4756, + "theoretical_loss": 3.721565251652142, + "tokens_seen": 816644096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800802407221665, + "loss": 2.5205, + "theoretical_loss": 3.7215357573352525, + "tokens_seen": 816709632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800702106318957, + "loss": 2.7038, + "theoretical_loss": 3.7215062660476352, + "tokens_seen": 816775168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800601805416249, + "loss": 2.499, + "theoretical_loss": 3.7214767777887356, + "tokens_seen": 816840704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038005015045135406, + "loss": 2.6442, + "theoretical_loss": 3.7214472925579996, + "tokens_seen": 816906240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038004012036108324, + "loss": 2.792, + "theoretical_loss": 3.7214178103548736, + "tokens_seen": 816971776 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800300902708124, + "loss": 2.8231, + "theoretical_loss": 3.7213883311788036, + "tokens_seen": 817037312 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038002006018054166, + "loss": 2.6325, + "theoretical_loss": 3.721358855029237, + "tokens_seen": 817102848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003800100300902708, + "loss": 2.4452, + "theoretical_loss": 3.721329381905619, + "tokens_seen": 817168384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00038, + "loss": 2.7884, + "theoretical_loss": 3.721299911807397, + "tokens_seen": 817233920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037998996990972915, + "loss": 2.6552, + "theoretical_loss": 3.721270444734018, + "tokens_seen": 817299456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799799398194584, + "loss": 2.8386, + "theoretical_loss": 3.7212409806849287, + "tokens_seen": 817364992 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037996990972918756, + "loss": 2.5461, + "theoretical_loss": 3.7212115196595765, + "tokens_seen": 817430528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037995987963891675, + "loss": 2.6774, + "theoretical_loss": 3.7211820616574087, + "tokens_seen": 817496064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 458869, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.828730344772339, + "objective/train/theoretical_loss": 3.7211526066778724, + "objective/train/tokens_used": 838021600, + "theoretical_loss": 3.7211526066778724, + "tokens_seen": 817561600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799498495486459, + "loss": 2.8248, + "theoretical_loss": 3.7211526066778724, + "tokens_seen": 817561600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037993981945837516, + "loss": 2.6308, + "theoretical_loss": 3.7211231547204155, + "tokens_seen": 817627136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799297893681043, + "loss": 2.7227, + "theoretical_loss": 3.7210937057844857, + "tokens_seen": 817692672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003799197592778335, + "loss": 2.4619, + "theoretical_loss": 3.7210642598695305, + "tokens_seen": 817758208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037990972918756265, + "loss": 2.5265, + "theoretical_loss": 3.7210348169749987, + "tokens_seen": 817823744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798996990972919, + "loss": 2.7976, + "theoretical_loss": 3.721005377100338, + "tokens_seen": 817889280 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037988966900702107, + "loss": 2.5068, + "theoretical_loss": 3.7209759402449967, + "tokens_seen": 817954816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037987963891675025, + "loss": 2.94, + "theoretical_loss": 3.7209465064084233, + "tokens_seen": 818020352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798696088264795, + "loss": 2.5955, + "theoretical_loss": 3.7209170755900667, + "tokens_seen": 818085888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798595787362086, + "loss": 2.8317, + "theoretical_loss": 3.7208876477893758, + "tokens_seen": 818151424 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037984954864593785, + "loss": 2.6954, + "theoretical_loss": 3.7208582230057985, + "tokens_seen": 818216960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037983951855566703, + "loss": 2.8328, + "theoretical_loss": 3.7208288012387847, + "tokens_seen": 818282496 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798294884653962, + "loss": 2.5533, + "theoretical_loss": 3.720799382487783, + "tokens_seen": 818348032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003798194583751254, + "loss": 2.7027, + "theoretical_loss": 3.7207699667522434, + "tokens_seen": 818413568 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037980942828485457, + "loss": 2.4817, + "theoretical_loss": 3.7207405540316154, + "tokens_seen": 818479104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037979939819458375, + "loss": 2.6257, + "theoretical_loss": 3.7207111443253478, + "tokens_seen": 818544640 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379789368104313, + "loss": 2.8737, + "theoretical_loss": 3.7206817376328907, + "tokens_seen": 818610176 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797793380140421, + "loss": 2.6199, + "theoretical_loss": 3.7206523339536943, + "tokens_seen": 818675712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037976930792377135, + "loss": 2.5395, + "theoretical_loss": 3.720622933287209, + "tokens_seen": 818741248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037975927783350053, + "loss": 2.6246, + "theoretical_loss": 3.7205935356328848, + "tokens_seen": 818806784 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797492477432297, + "loss": 2.797, + "theoretical_loss": 3.720564140990171, + "tokens_seen": 818872320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797392176529589, + "loss": 2.7855, + "theoretical_loss": 3.720534749358519, + "tokens_seen": 818937856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797291875626881, + "loss": 2.7592, + "theoretical_loss": 3.7205053607373797, + "tokens_seen": 819003392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037971915747241725, + "loss": 2.515, + "theoretical_loss": 3.720475975126203, + "tokens_seen": 819068928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003797091273821465, + "loss": 2.4283, + "theoretical_loss": 3.72044659252444, + "tokens_seen": 819134464 + }, + { + "debugging/Self-BLEU-5": 0.4343450632679149, + "debugging/distinct-1-grams": 0.7759737985147822, + "debugging/distinct-2-grams": 0.9682119066950529, + "debugging/entropy-1-grams": 5.4098389736484584, + "debugging/entropy-2-grams": 6.131371694974163, + "debugging/length": 546.8333333333334, + "debugging/num_segments": 6, + "epoch": 0.25, + "objective/train/docs_used": 459733, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.788059711456299, + "objective/train/theoretical_loss": 3.720417212931543, + "objective/train/tokens_used": 839660000, + "theoretical_loss": 3.720417212931543, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796990972918756, + "loss": 2.7732, + "theoretical_loss": 3.720417212931543, + "tokens_seen": 819200000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037968906720160485, + "loss": 2.6896, + "theoretical_loss": 3.720387836346962, + "tokens_seen": 819265536 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379679037111334, + "loss": 2.5718, + "theoretical_loss": 3.7203584627701485, + "tokens_seen": 819331072 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796690070210632, + "loss": 2.7729, + "theoretical_loss": 3.7203290922005543, + "tokens_seen": 819396608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796589769307924, + "loss": 2.7949, + "theoretical_loss": 3.7202997246376306, + "tokens_seen": 819462144 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796489468405216, + "loss": 2.7875, + "theoretical_loss": 3.72027036008083, + "tokens_seen": 819527680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037963891675025076, + "loss": 2.5432, + "theoretical_loss": 3.7202409985296034, + "tokens_seen": 819593216 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037962888665997994, + "loss": 2.8756, + "theoretical_loss": 3.7202116399834035, + "tokens_seen": 819658752 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003796188565697091, + "loss": 2.6533, + "theoretical_loss": 3.720182284441682, + "tokens_seen": 819724288 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037960882647943836, + "loss": 2.8107, + "theoretical_loss": 3.7201529319038924, + "tokens_seen": 819789824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795987963891675, + "loss": 2.8196, + "theoretical_loss": 3.720123582369486, + "tokens_seen": 819855360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795887662988967, + "loss": 2.8017, + "theoretical_loss": 3.720094235837916, + "tokens_seen": 819920896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795787362086259, + "loss": 2.7922, + "theoretical_loss": 3.7200648923086357, + "tokens_seen": 819986432 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795687061183551, + "loss": 2.5626, + "theoretical_loss": 3.7200355517810966, + "tokens_seen": 820051968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037955867602808426, + "loss": 2.4739, + "theoretical_loss": 3.7200062142547528, + "tokens_seen": 820117504 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037954864593781344, + "loss": 2.7152, + "theoretical_loss": 3.7199768797290576, + "tokens_seen": 820183040 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795386158475426, + "loss": 2.8596, + "theoretical_loss": 3.7199475482034643, + "tokens_seen": 820248576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037952858575727186, + "loss": 2.817, + "theoretical_loss": 3.7199182196774254, + "tokens_seen": 820314112 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379518555667001, + "loss": 2.5966, + "theoretical_loss": 3.719888894150396, + "tokens_seen": 820379648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003795085255767302, + "loss": 2.8667, + "theoretical_loss": 3.7198595716218295, + "tokens_seen": 820445184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037949849548645935, + "loss": 2.5299, + "theoretical_loss": 3.719830252091179, + "tokens_seen": 820510720 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794884653961886, + "loss": 2.4508, + "theoretical_loss": 3.7198009355578994, + "tokens_seen": 820576256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037947843530591776, + "loss": 2.7271, + "theoretical_loss": 3.7197716220214443, + "tokens_seen": 820641792 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037946840521564695, + "loss": 2.6313, + "theoretical_loss": 3.7197423114812693, + "tokens_seen": 820707328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794583751253761, + "loss": 2.8888, + "theoretical_loss": 3.7197130039368274, + "tokens_seen": 820772864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 460372, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5753989219665527, + "objective/train/theoretical_loss": 3.7196836993875744, + "objective/train/tokens_used": 841298400, + "theoretical_loss": 3.7196836993875744, + "tokens_seen": 820838400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037944834503510536, + "loss": 2.6251, + "theoretical_loss": 3.7196836993875744, + "tokens_seen": 820838400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794383149448345, + "loss": 2.6649, + "theoretical_loss": 3.719654397832964, + "tokens_seen": 820903936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794282848545637, + "loss": 2.6335, + "theoretical_loss": 3.7196250992724527, + "tokens_seen": 820969472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037941825476429285, + "loss": 2.7872, + "theoretical_loss": 3.719595803705494, + "tokens_seen": 821035008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003794082246740221, + "loss": 2.7039, + "theoretical_loss": 3.7195665111315437, + "tokens_seen": 821100544 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037939819458375127, + "loss": 2.5931, + "theoretical_loss": 3.7195372215500573, + "tokens_seen": 821166080 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037938816449348045, + "loss": 2.4618, + "theoretical_loss": 3.7195079349604905, + "tokens_seen": 821231616 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037937813440320963, + "loss": 2.5012, + "theoretical_loss": 3.719478651362299, + "tokens_seen": 821297152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793681043129388, + "loss": 2.6347, + "theoretical_loss": 3.719449370754938, + "tokens_seen": 821362688 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379358074222668, + "loss": 2.5941, + "theoretical_loss": 3.7194200931378636, + "tokens_seen": 821428224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037934804413239723, + "loss": 2.6462, + "theoretical_loss": 3.719390818510532, + "tokens_seen": 821493760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037933801404212635, + "loss": 2.7512, + "theoretical_loss": 3.7193615468724, + "tokens_seen": 821559296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793279839518556, + "loss": 2.7186, + "theoretical_loss": 3.7193322782229226, + "tokens_seen": 821624832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003793179538615847, + "loss": 2.7558, + "theoretical_loss": 3.719303012561557, + "tokens_seen": 821690368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037930792377131395, + "loss": 2.676, + "theoretical_loss": 3.719273749887761, + "tokens_seen": 821755904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037929789368104313, + "loss": 2.5643, + "theoretical_loss": 3.71924449020099, + "tokens_seen": 821821440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792878635907723, + "loss": 2.5705, + "theoretical_loss": 3.7192152335007007, + "tokens_seen": 821886976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792778335005015, + "loss": 2.8495, + "theoretical_loss": 3.719185979786351, + "tokens_seen": 821952512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037926780341023073, + "loss": 2.8069, + "theoretical_loss": 3.719156729057398, + "tokens_seen": 822018048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037925777331995986, + "loss": 2.6097, + "theoretical_loss": 3.719127481313299, + "tokens_seen": 822083584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792477432296891, + "loss": 2.5657, + "theoretical_loss": 3.719098236553511, + "tokens_seen": 822149120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792377131394182, + "loss": 2.7598, + "theoretical_loss": 3.7190689947774924, + "tokens_seen": 822214656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037922768304914746, + "loss": 2.8197, + "theoretical_loss": 3.7190397559847006, + "tokens_seen": 822280192 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037921765295887664, + "loss": 2.7267, + "theoretical_loss": 3.7190105201745935, + "tokens_seen": 822345728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003792076228686058, + "loss": 2.8076, + "theoretical_loss": 3.7189812873466295, + "tokens_seen": 822411264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 461421, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8889474868774414, + "objective/train/theoretical_loss": 3.7189520575002666, + "objective/train/tokens_used": 842936800, + "theoretical_loss": 3.7189520575002666, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.000379197592778335, + "loss": 2.8099, + "theoretical_loss": 3.7189520575002666, + "tokens_seen": 822476800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791875626880642, + "loss": 2.6691, + "theoretical_loss": 3.7189228306349627, + "tokens_seen": 822542336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037917753259779336, + "loss": 2.6943, + "theoretical_loss": 3.7188936067501768, + "tokens_seen": 822607872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791675025075226, + "loss": 2.8235, + "theoretical_loss": 3.718864385845367, + "tokens_seen": 822673408 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791574724172517, + "loss": 2.7914, + "theoretical_loss": 3.7188351679199934, + "tokens_seen": 822738944 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037914744232698096, + "loss": 2.7442, + "theoretical_loss": 3.7188059529735136, + "tokens_seen": 822804480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791374122367101, + "loss": 2.7754, + "theoretical_loss": 3.7187767410053874, + "tokens_seen": 822870016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791273821464393, + "loss": 2.5717, + "theoretical_loss": 3.7187475320150734, + "tokens_seen": 822935552 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037911735205616856, + "loss": 2.7173, + "theoretical_loss": 3.7187183260020307, + "tokens_seen": 823001088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003791073219658977, + "loss": 2.8607, + "theoretical_loss": 3.7186891229657197, + "tokens_seen": 823066624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790972918756269, + "loss": 2.7677, + "theoretical_loss": 3.7186599229056, + "tokens_seen": 823132160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790872617853561, + "loss": 2.8683, + "theoretical_loss": 3.718630725821131, + "tokens_seen": 823197696 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790772316950853, + "loss": 2.6781, + "theoretical_loss": 3.718601531711772, + "tokens_seen": 823263232 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037906720160481446, + "loss": 2.7821, + "theoretical_loss": 3.718572340576984, + "tokens_seen": 823328768 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037905717151454364, + "loss": 2.2804, + "theoretical_loss": 3.7185431524162267, + "tokens_seen": 823394304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790471414242728, + "loss": 2.5982, + "theoretical_loss": 3.7185139672289607, + "tokens_seen": 823459840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037903711133400206, + "loss": 2.7199, + "theoretical_loss": 3.718484785014646, + "tokens_seen": 823525376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790270812437312, + "loss": 2.814, + "theoretical_loss": 3.7184556057727436, + "tokens_seen": 823590912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003790170511534604, + "loss": 2.4827, + "theoretical_loss": 3.7184264295027143, + "tokens_seen": 823656448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037900702106318955, + "loss": 2.7661, + "theoretical_loss": 3.718397256204019, + "tokens_seen": 823721984 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789969909729188, + "loss": 2.8633, + "theoretical_loss": 3.7183680858761186, + "tokens_seen": 823787520 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037898696088264796, + "loss": 2.8532, + "theoretical_loss": 3.718338918518474, + "tokens_seen": 823853056 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037897693079237715, + "loss": 2.5904, + "theoretical_loss": 3.718309754130547, + "tokens_seen": 823918592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789669007021063, + "loss": 2.6954, + "theoretical_loss": 3.718280592711799, + "tokens_seen": 823984128 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037895687061183556, + "loss": 2.9142, + "theoretical_loss": 3.7182514342616906, + "tokens_seen": 824049664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 462012, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.056985855102539, + "objective/train/theoretical_loss": 3.7182222787796846, + "objective/train/tokens_used": 844575200, + "theoretical_loss": 3.7182222787796846, + "tokens_seen": 824115200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789468405215647, + "loss": 2.7643, + "theoretical_loss": 3.7182222787796846, + "tokens_seen": 824115200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789368104312939, + "loss": 2.7109, + "theoretical_loss": 3.718193126265243, + "tokens_seen": 824180736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037892678034102305, + "loss": 2.6935, + "theoretical_loss": 3.718163976717828, + "tokens_seen": 824246272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003789167502507523, + "loss": 2.5794, + "theoretical_loss": 3.7181348301369006, + "tokens_seen": 824311808 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037890672016048147, + "loss": 2.6364, + "theoretical_loss": 3.7181056865219233, + "tokens_seen": 824377344 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037889669007021065, + "loss": 2.678, + "theoretical_loss": 3.7180765458723597, + "tokens_seen": 824442880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037888665997993983, + "loss": 2.6142, + "theoretical_loss": 3.718047408187671, + "tokens_seen": 824508416 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378876629889669, + "loss": 2.6126, + "theoretical_loss": 3.7180182734673206, + "tokens_seen": 824573952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788665997993982, + "loss": 2.3591, + "theoretical_loss": 3.7179891417107718, + "tokens_seen": 824639488 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037885656970912743, + "loss": 2.6444, + "theoretical_loss": 3.717960012917487, + "tokens_seen": 824705024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037884653961885655, + "loss": 2.842, + "theoretical_loss": 3.717930887086929, + "tokens_seen": 824770560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788365095285858, + "loss": 2.8211, + "theoretical_loss": 3.7179017642185617, + "tokens_seen": 824836096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003788264794383149, + "loss": 2.7662, + "theoretical_loss": 3.717872644311848, + "tokens_seen": 824901632 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037881644934804415, + "loss": 2.6669, + "theoretical_loss": 3.7178435273662522, + "tokens_seen": 824967168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037880641925777333, + "loss": 2.953, + "theoretical_loss": 3.7178144133812374, + "tokens_seen": 825032704 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787963891675025, + "loss": 2.6334, + "theoretical_loss": 3.7177853023562673, + "tokens_seen": 825098240 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787863590772317, + "loss": 2.847, + "theoretical_loss": 3.7177561942908066, + "tokens_seen": 825163776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037877632898696093, + "loss": 2.6949, + "theoretical_loss": 3.717727089184319, + "tokens_seen": 825229312 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037876629889669006, + "loss": 2.6026, + "theoretical_loss": 3.717697987036269, + "tokens_seen": 825294848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787562688064193, + "loss": 2.9019, + "theoretical_loss": 3.71766888784612, + "tokens_seen": 825360384 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787462387161484, + "loss": 2.733, + "theoretical_loss": 3.7176397916133377, + "tokens_seen": 825425920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037873620862587766, + "loss": 2.5797, + "theoretical_loss": 3.7176106983373858, + "tokens_seen": 825491456 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037872617853560684, + "loss": 2.6358, + "theoretical_loss": 3.7175816080177304, + "tokens_seen": 825556992 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378716148445336, + "loss": 2.93, + "theoretical_loss": 3.717552520653835, + "tokens_seen": 825622528 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003787061183550652, + "loss": 2.7954, + "theoretical_loss": 3.717523436245166, + "tokens_seen": 825688064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 463076, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.083082914352417, + "objective/train/theoretical_loss": 3.717494354791188, + "objective/train/tokens_used": 846213600, + "theoretical_loss": 3.717494354791188, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786960882647944, + "loss": 3.0909, + "theoretical_loss": 3.717494354791188, + "tokens_seen": 825753600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037868605817452356, + "loss": 3.0866, + "theoretical_loss": 3.7174652762913656, + "tokens_seen": 825819136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786760280842528, + "loss": 2.7781, + "theoretical_loss": 3.7174362007451656, + "tokens_seen": 825884672 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786659979939819, + "loss": 2.5382, + "theoretical_loss": 3.717407128152053, + "tokens_seen": 825950208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037865596790371116, + "loss": 2.5681, + "theoretical_loss": 3.7173780585114944, + "tokens_seen": 826015744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786459378134403, + "loss": 2.6316, + "theoretical_loss": 3.7173489918229548, + "tokens_seen": 826081280 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786359077231695, + "loss": 2.6745, + "theoretical_loss": 3.7173199280859, + "tokens_seen": 826146816 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786258776328987, + "loss": 2.7971, + "theoretical_loss": 3.717290867299797, + "tokens_seen": 826212352 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003786158475426279, + "loss": 2.6546, + "theoretical_loss": 3.717261809464112, + "tokens_seen": 826277888 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037860581745235706, + "loss": 2.5943, + "theoretical_loss": 3.717232754578311, + "tokens_seen": 826343424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785957873620863, + "loss": 2.8337, + "theoretical_loss": 3.717203702641861, + "tokens_seen": 826408960 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785857572718154, + "loss": 2.8984, + "theoretical_loss": 3.717174653654229, + "tokens_seen": 826474496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037857572718154466, + "loss": 2.8819, + "theoretical_loss": 3.7171456076148814, + "tokens_seen": 826540032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785656970912738, + "loss": 2.6567, + "theoretical_loss": 3.7171165645232858, + "tokens_seen": 826605568 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378555667001003, + "loss": 2.8761, + "theoretical_loss": 3.7170875243789085, + "tokens_seen": 826671104 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785456369107322, + "loss": 2.8024, + "theoretical_loss": 3.717058487181218, + "tokens_seen": 826736640 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003785356068204614, + "loss": 2.7838, + "theoretical_loss": 3.7170294529296806, + "tokens_seen": 826802176 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037852557673019057, + "loss": 2.7053, + "theoretical_loss": 3.7170004216237644, + "tokens_seen": 826867712 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037851554663991975, + "loss": 2.8404, + "theoretical_loss": 3.716971393262937, + "tokens_seen": 826933248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037850551654964893, + "loss": 2.7194, + "theoretical_loss": 3.7169423678466664, + "tokens_seen": 826998784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037849548645937816, + "loss": 2.8596, + "theoretical_loss": 3.7169133453744205, + "tokens_seen": 827064320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784854563691073, + "loss": 2.8185, + "theoretical_loss": 3.716884325845668, + "tokens_seen": 827129856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784754262788365, + "loss": 2.5357, + "theoretical_loss": 3.716855309259877, + "tokens_seen": 827195392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037846539618856565, + "loss": 2.6472, + "theoretical_loss": 3.7168262956165146, + "tokens_seen": 827260928 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784553660982949, + "loss": 2.6843, + "theoretical_loss": 3.7167972849150512, + "tokens_seen": 827326464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 464152, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.622303009033203, + "objective/train/theoretical_loss": 3.716768277154954, + "objective/train/tokens_used": 847852000, + "theoretical_loss": 3.716768277154954, + "tokens_seen": 827392000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037844533600802407, + "loss": 2.6362, + "theoretical_loss": 3.716768277154954, + "tokens_seen": 827392000 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037843530591775325, + "loss": 2.7811, + "theoretical_loss": 3.716739272335693, + "tokens_seen": 827457536 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037842527582748243, + "loss": 2.7028, + "theoretical_loss": 3.7167102704567365, + "tokens_seen": 827523072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037841524573721167, + "loss": 2.8274, + "theoretical_loss": 3.7166812715175537, + "tokens_seen": 827588608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003784052156469408, + "loss": 2.7487, + "theoretical_loss": 3.716652275517614, + "tokens_seen": 827654144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037839518555667003, + "loss": 2.7647, + "theoretical_loss": 3.716623282456387, + "tokens_seen": 827719680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037838515546639916, + "loss": 2.9639, + "theoretical_loss": 3.7165942923333417, + "tokens_seen": 827785216 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783751253761284, + "loss": 2.8212, + "theoretical_loss": 3.716565305147948, + "tokens_seen": 827850752 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037836509528585763, + "loss": 2.4917, + "theoretical_loss": 3.7165363208996753, + "tokens_seen": 827916288 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037835506519558675, + "loss": 2.9519, + "theoretical_loss": 3.716507339587995, + "tokens_seen": 827981824 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378345035105316, + "loss": 2.9141, + "theoretical_loss": 3.716478361212375, + "tokens_seen": 828047360 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783350050150451, + "loss": 2.7782, + "theoretical_loss": 3.716449385772288, + "tokens_seen": 828112896 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037832497492477435, + "loss": 2.6236, + "theoretical_loss": 3.716420413267201, + "tokens_seen": 828178432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037831494483450353, + "loss": 2.834, + "theoretical_loss": 3.7163914436965877, + "tokens_seen": 828243968 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003783049147442327, + "loss": 2.597, + "theoretical_loss": 3.7163624770599175, + "tokens_seen": 828309504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782948846539619, + "loss": 2.7529, + "theoretical_loss": 3.716333513356661, + "tokens_seen": 828375040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037828485456369113, + "loss": 2.3636, + "theoretical_loss": 3.7163045525862897, + "tokens_seen": 828440576 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037827482447342026, + "loss": 2.833, + "theoretical_loss": 3.7162755947482733, + "tokens_seen": 828506112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782647943831495, + "loss": 2.7211, + "theoretical_loss": 3.716246639842084, + "tokens_seen": 828571648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782547642928786, + "loss": 2.7162, + "theoretical_loss": 3.7162176878671938, + "tokens_seen": 828637184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037824473420260786, + "loss": 2.7256, + "theoretical_loss": 3.7161887388230728, + "tokens_seen": 828702720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037823470411233704, + "loss": 2.9198, + "theoretical_loss": 3.7161597927091927, + "tokens_seen": 828768256 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782246740220662, + "loss": 2.8702, + "theoretical_loss": 3.716130849525026, + "tokens_seen": 828833792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782146439317954, + "loss": 2.7121, + "theoretical_loss": 3.7161019092700442, + "tokens_seen": 828899328 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003782046138415246, + "loss": 2.5721, + "theoretical_loss": 3.716072971943719, + "tokens_seen": 828964864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 464802, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.640942096710205, + "objective/train/theoretical_loss": 3.716044037545523, + "objective/train/tokens_used": 849490400, + "theoretical_loss": 3.716044037545523, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037819458375125376, + "loss": 2.6406, + "theoretical_loss": 3.716044037545523, + "tokens_seen": 829030400 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378184553660983, + "loss": 2.7612, + "theoretical_loss": 3.716015106074928, + "tokens_seen": 829095936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781745235707121, + "loss": 2.5001, + "theoretical_loss": 3.715986177531407, + "tokens_seen": 829161472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037816449348044136, + "loss": 2.8508, + "theoretical_loss": 3.715957251914432, + "tokens_seen": 829227008 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781544633901705, + "loss": 2.9323, + "theoretical_loss": 3.715928329223476, + "tokens_seen": 829292544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781444332998997, + "loss": 2.7472, + "theoretical_loss": 3.715899409458012, + "tokens_seen": 829358080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781344032096289, + "loss": 2.8221, + "theoretical_loss": 3.7158704926175123, + "tokens_seen": 829423616 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781243731193581, + "loss": 2.7649, + "theoretical_loss": 3.7158415787014505, + "tokens_seen": 829489152 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037811434302908726, + "loss": 2.7626, + "theoretical_loss": 3.7158126677092995, + "tokens_seen": 829554688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003781043129388165, + "loss": 2.777, + "theoretical_loss": 3.7157837596405328, + "tokens_seen": 829620224 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780942828485456, + "loss": 2.8599, + "theoretical_loss": 3.7157548544946244, + "tokens_seen": 829685760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037808425275827486, + "loss": 2.7063, + "theoretical_loss": 3.7157259522710473, + "tokens_seen": 829751296 + }, + { + "epoch": 0.25, + "learning_rate": 0.000378074222668004, + "loss": 2.7576, + "theoretical_loss": 3.7156970529692757, + "tokens_seen": 829816832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780641925777332, + "loss": 2.6135, + "theoretical_loss": 3.7156681565887824, + "tokens_seen": 829882368 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780541624874624, + "loss": 2.6237, + "theoretical_loss": 3.715639263129043, + "tokens_seen": 829947904 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003780441323971916, + "loss": 2.6227, + "theoretical_loss": 3.7156103725895306, + "tokens_seen": 830013440 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037803410230692077, + "loss": 2.5027, + "theoretical_loss": 3.71558148496972, + "tokens_seen": 830078976 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037802407221664995, + "loss": 2.6977, + "theoretical_loss": 3.7155526002690857, + "tokens_seen": 830144512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037801404212637913, + "loss": 2.5946, + "theoretical_loss": 3.715523718487102, + "tokens_seen": 830210048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037800401203610836, + "loss": 2.7186, + "theoretical_loss": 3.7154948396232443, + "tokens_seen": 830275584 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003779939819458375, + "loss": 2.7893, + "theoretical_loss": 3.7154659636769862, + "tokens_seen": 830341120 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037798395185556673, + "loss": 2.7367, + "theoretical_loss": 3.7154370906478036, + "tokens_seen": 830406656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037797392176529585, + "loss": 2.6869, + "theoretical_loss": 3.7154082205351715, + "tokens_seen": 830472192 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003779638916750251, + "loss": 2.8705, + "theoretical_loss": 3.715379353338565, + "tokens_seen": 830537728 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037795386158475427, + "loss": 2.7076, + "theoretical_loss": 3.7153504890574602, + "tokens_seen": 830603264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 465950, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4607279300689697, + "objective/train/theoretical_loss": 3.7153216276913312, + "objective/train/tokens_used": 851128800, + "theoretical_loss": 3.7153216276913312, + "tokens_seen": 830668800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037794383149448345, + "loss": 2.4907, + "theoretical_loss": 3.7153216276913312, + "tokens_seen": 830668800 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037793380140421263, + "loss": 2.7686, + "theoretical_loss": 3.7152927692396553, + "tokens_seen": 830734336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037792377131394187, + "loss": 2.9923, + "theoretical_loss": 3.7152639137019072, + "tokens_seen": 830799872 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377913741223671, + "loss": 2.7894, + "theoretical_loss": 3.7152350610775633, + "tokens_seen": 830865408 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037790371113340023, + "loss": 2.3556, + "theoretical_loss": 3.7152062113660995, + "tokens_seen": 830930944 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037789368104312936, + "loss": 2.7755, + "theoretical_loss": 3.7151773645669923, + "tokens_seen": 830996480 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778836509528586, + "loss": 2.7287, + "theoretical_loss": 3.7151485206797177, + "tokens_seen": 831062016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778736208625878, + "loss": 2.8398, + "theoretical_loss": 3.7151196797037525, + "tokens_seen": 831127552 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037786359077231695, + "loss": 2.686, + "theoretical_loss": 3.715090841638573, + "tokens_seen": 831193088 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037785356068204614, + "loss": 2.6514, + "theoretical_loss": 3.715062006483656, + "tokens_seen": 831258624 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778435305917753, + "loss": 2.8311, + "theoretical_loss": 3.7150331742384783, + "tokens_seen": 831324160 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778335005015045, + "loss": 2.819, + "theoretical_loss": 3.7150043449025176, + "tokens_seen": 831389696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037782347041123373, + "loss": 2.8208, + "theoretical_loss": 3.71497551847525, + "tokens_seen": 831455232 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037781344032096286, + "loss": 2.6147, + "theoretical_loss": 3.7149466949561547, + "tokens_seen": 831520768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003778034102306921, + "loss": 2.6827, + "theoretical_loss": 3.7149178743447067, + "tokens_seen": 831586304 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777933801404213, + "loss": 2.7768, + "theoretical_loss": 3.714889056640385, + "tokens_seen": 831651840 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037778335005015046, + "loss": 2.4861, + "theoretical_loss": 3.7148602418426675, + "tokens_seen": 831717376 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037777331995987964, + "loss": 2.6438, + "theoretical_loss": 3.7148314299510314, + "tokens_seen": 831782912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777632898696088, + "loss": 2.8952, + "theoretical_loss": 3.7148026209649543, + "tokens_seen": 831848448 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377753259779338, + "loss": 2.7831, + "theoretical_loss": 3.714773814883915, + "tokens_seen": 831913984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037774322968906724, + "loss": 2.751, + "theoretical_loss": 3.714745011707392, + "tokens_seen": 831979520 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037773319959879636, + "loss": 2.5419, + "theoretical_loss": 3.714716211434863, + "tokens_seen": 832045056 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777231695085256, + "loss": 2.954, + "theoretical_loss": 3.714687414065807, + "tokens_seen": 832110592 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003777131394182547, + "loss": 2.6081, + "theoretical_loss": 3.7146586195997022, + "tokens_seen": 832176128 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037770310932798396, + "loss": 2.8164, + "theoretical_loss": 3.7146298280360277, + "tokens_seen": 832241664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 466904, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.74540638923645, + "objective/train/theoretical_loss": 3.714601039374263, + "objective/train/tokens_used": 852767200, + "theoretical_loss": 3.714601039374263, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037769307923771314, + "loss": 2.7948, + "theoretical_loss": 3.714601039374263, + "tokens_seen": 832307200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776830491474423, + "loss": 2.7842, + "theoretical_loss": 3.714572253613886, + "tokens_seen": 832372736 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776730190571715, + "loss": 2.7081, + "theoretical_loss": 3.714543470754376, + "tokens_seen": 832438272 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776629889669007, + "loss": 2.7968, + "theoretical_loss": 3.714514690795213, + "tokens_seen": 832503808 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037765295887662987, + "loss": 2.92, + "theoretical_loss": 3.714485913735876, + "tokens_seen": 832569344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776429287863591, + "loss": 2.6305, + "theoretical_loss": 3.7144571395758454, + "tokens_seen": 832634880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037763289869608823, + "loss": 2.8311, + "theoretical_loss": 3.7144283683145995, + "tokens_seen": 832700416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037762286860581746, + "loss": 2.9982, + "theoretical_loss": 3.7143995999516193, + "tokens_seen": 832765952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776128385155467, + "loss": 2.915, + "theoretical_loss": 3.7143708344863846, + "tokens_seen": 832831488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003776028084252758, + "loss": 2.6645, + "theoretical_loss": 3.7143420719183755, + "tokens_seen": 832897024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037759277833500506, + "loss": 2.8094, + "theoretical_loss": 3.7143133122470715, + "tokens_seen": 832962560 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775827482447342, + "loss": 2.8851, + "theoretical_loss": 3.7142845554719544, + "tokens_seen": 833028096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775727181544634, + "loss": 2.6911, + "theoretical_loss": 3.714255801592503, + "tokens_seen": 833093632 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775626880641926, + "loss": 2.7356, + "theoretical_loss": 3.7142270506081996, + "tokens_seen": 833159168 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775526579739218, + "loss": 2.9705, + "theoretical_loss": 3.7141983025185246, + "tokens_seen": 833224704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037754262788365097, + "loss": 2.901, + "theoretical_loss": 3.714169557322958, + "tokens_seen": 833290240 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037753259779338015, + "loss": 2.9581, + "theoretical_loss": 3.714140815020982, + "tokens_seen": 833355776 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037752256770310933, + "loss": 2.7036, + "theoretical_loss": 3.7141120756120767, + "tokens_seen": 833421312 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037751253761283857, + "loss": 2.7045, + "theoretical_loss": 3.714083339095725, + "tokens_seen": 833486848 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003775025075225677, + "loss": 2.8971, + "theoretical_loss": 3.7140546054714068, + "tokens_seen": 833552384 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037749247743229693, + "loss": 2.6651, + "theoretical_loss": 3.7140258747386046, + "tokens_seen": 833617920 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037748244734202605, + "loss": 2.7434, + "theoretical_loss": 3.7139971468967996, + "tokens_seen": 833683456 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774724172517553, + "loss": 2.8242, + "theoretical_loss": 3.713968421945474, + "tokens_seen": 833748992 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037746238716148447, + "loss": 2.7712, + "theoretical_loss": 3.7139396998841097, + "tokens_seen": 833814528 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037745235707121365, + "loss": 2.8285, + "theoretical_loss": 3.713910980712189, + "tokens_seen": 833880064 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 467427, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3416881561279297, + "objective/train/theoretical_loss": 3.713882264429194, + "objective/train/tokens_used": 854405600, + "theoretical_loss": 3.713882264429194, + "tokens_seen": 833945600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037744232698094283, + "loss": 2.7444, + "theoretical_loss": 3.713882264429194, + "tokens_seen": 833945600 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037743229689067207, + "loss": 2.7785, + "theoretical_loss": 3.713853551034607, + "tokens_seen": 834011136 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003774222668004012, + "loss": 2.7879, + "theoretical_loss": 3.713824840527911, + "tokens_seen": 834076672 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037741223671013043, + "loss": 2.8718, + "theoretical_loss": 3.7137961329085885, + "tokens_seen": 834142208 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037740220661985956, + "loss": 2.618, + "theoretical_loss": 3.713767428176122, + "tokens_seen": 834207744 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773921765295888, + "loss": 2.6378, + "theoretical_loss": 3.713738726329994, + "tokens_seen": 834273280 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377382146439318, + "loss": 2.7106, + "theoretical_loss": 3.7137100273696886, + "tokens_seen": 834338816 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037737211634904715, + "loss": 2.7905, + "theoretical_loss": 3.713681331294689, + "tokens_seen": 834404352 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037736208625877634, + "loss": 2.8341, + "theoretical_loss": 3.7136526381044774, + "tokens_seen": 834469888 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773520561685055, + "loss": 3.0653, + "theoretical_loss": 3.7136239477985384, + "tokens_seen": 834535424 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773420260782347, + "loss": 2.6993, + "theoretical_loss": 3.713595260376355, + "tokens_seen": 834600960 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037733199598796393, + "loss": 2.6912, + "theoretical_loss": 3.713566575837411, + "tokens_seen": 834666496 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037732196589769306, + "loss": 2.887, + "theoretical_loss": 3.713537894181191, + "tokens_seen": 834732032 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773119358074223, + "loss": 2.8895, + "theoretical_loss": 3.7135092154071776, + "tokens_seen": 834797568 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003773019057171515, + "loss": 2.737, + "theoretical_loss": 3.7134805395148556, + "tokens_seen": 834863104 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037729187562688066, + "loss": 2.5348, + "theoretical_loss": 3.71345186650371, + "tokens_seen": 834928640 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037728184553660984, + "loss": 2.673, + "theoretical_loss": 3.7134231963732245, + "tokens_seen": 834994176 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377271815446339, + "loss": 2.5236, + "theoretical_loss": 3.7133945291228834, + "tokens_seen": 835059712 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772617853560682, + "loss": 2.8603, + "theoretical_loss": 3.713365864752172, + "tokens_seen": 835125248 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037725175526579744, + "loss": 2.7683, + "theoretical_loss": 3.7133372032605747, + "tokens_seen": 835190784 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037724172517552656, + "loss": 2.822, + "theoretical_loss": 3.7133085446475764, + "tokens_seen": 835256320 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772316950852558, + "loss": 2.6931, + "theoretical_loss": 3.7132798889126626, + "tokens_seen": 835321856 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003772216649949849, + "loss": 2.5958, + "theoretical_loss": 3.713251236055317, + "tokens_seen": 835387392 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037721163490471416, + "loss": 2.7673, + "theoretical_loss": 3.7132225860750276, + "tokens_seen": 835452928 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037720160481444334, + "loss": 2.8958, + "theoretical_loss": 3.713193938971278, + "tokens_seen": 835518464 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 468837, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6179723739624023, + "objective/train/theoretical_loss": 3.7131652947435536, + "objective/train/tokens_used": 856044000, + "theoretical_loss": 3.7131652947435536, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771915747241725, + "loss": 2.513, + "theoretical_loss": 3.7131652947435536, + "tokens_seen": 835584000 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771815446339017, + "loss": 2.6243, + "theoretical_loss": 3.7131366533913406, + "tokens_seen": 835649536 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771715145436309, + "loss": 2.6509, + "theoretical_loss": 3.713108014914125, + "tokens_seen": 835715072 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037716148445336007, + "loss": 2.594, + "theoretical_loss": 3.7130793793113934, + "tokens_seen": 835780608 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771514543630893, + "loss": 2.697, + "theoretical_loss": 3.7130507465826303, + "tokens_seen": 835846144 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037714142427281843, + "loss": 2.6427, + "theoretical_loss": 3.713022116727324, + "tokens_seen": 835911680 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037713139418254766, + "loss": 2.9435, + "theoretical_loss": 3.7129934897449592, + "tokens_seen": 835977216 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037712136409227685, + "loss": 2.7768, + "theoretical_loss": 3.7129648656350227, + "tokens_seen": 836042752 + }, + { + "epoch": 0.25, + "learning_rate": 0.000377111334002006, + "loss": 2.7805, + "theoretical_loss": 3.7129362443970013, + "tokens_seen": 836108288 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003771013039117352, + "loss": 2.5428, + "theoretical_loss": 3.7129076260303826, + "tokens_seen": 836173824 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770912738214644, + "loss": 2.8572, + "theoretical_loss": 3.7128790105346523, + "tokens_seen": 836239360 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037708124373119357, + "loss": 2.6759, + "theoretical_loss": 3.7128503979092984, + "tokens_seen": 836304896 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770712136409228, + "loss": 2.9712, + "theoretical_loss": 3.7128217881538075, + "tokens_seen": 836370432 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037706118355065193, + "loss": 2.9666, + "theoretical_loss": 3.712793181267667, + "tokens_seen": 836435968 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037705115346038117, + "loss": 2.6751, + "theoretical_loss": 3.7127645772503644, + "tokens_seen": 836501504 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770411233701103, + "loss": 2.9037, + "theoretical_loss": 3.7127359761013876, + "tokens_seen": 836567040 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037703109327983953, + "loss": 2.8934, + "theoretical_loss": 3.7127073778202235, + "tokens_seen": 836632576 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770210631895687, + "loss": 2.7859, + "theoretical_loss": 3.7126787824063605, + "tokens_seen": 836698112 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770110330992979, + "loss": 2.5454, + "theoretical_loss": 3.7126501898592865, + "tokens_seen": 836763648 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003770010030090271, + "loss": 2.836, + "theoretical_loss": 3.71262160017849, + "tokens_seen": 836829184 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037699097291875625, + "loss": 2.8422, + "theoretical_loss": 3.7125930133634584, + "tokens_seen": 836894720 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037698094282848544, + "loss": 2.8695, + "theoretical_loss": 3.712564429413681, + "tokens_seen": 836960256 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037697091273821467, + "loss": 2.9816, + "theoretical_loss": 3.7125358483286455, + "tokens_seen": 837025792 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769608826479438, + "loss": 2.7577, + "theoretical_loss": 3.71250727010784, + "tokens_seen": 837091328 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037695085255767303, + "loss": 2.8082, + "theoretical_loss": 3.7124786947507546, + "tokens_seen": 837156864 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 470129, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7207224369049072, + "objective/train/theoretical_loss": 3.712450122256878, + "objective/train/tokens_used": 857682400, + "theoretical_loss": 3.712450122256878, + "tokens_seen": 837222400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769408224674022, + "loss": 2.7488, + "theoretical_loss": 3.712450122256878, + "tokens_seen": 837222400 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769307923771314, + "loss": 2.6886, + "theoretical_loss": 3.712421552625699, + "tokens_seen": 837287936 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003769207622868606, + "loss": 2.4888, + "theoretical_loss": 3.712392985856706, + "tokens_seen": 837353472 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037691073219658976, + "loss": 2.9448, + "theoretical_loss": 3.712364421949389, + "tokens_seen": 837419008 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037690070210631894, + "loss": 2.7908, + "theoretical_loss": 3.712335860903238, + "tokens_seen": 837484544 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768906720160482, + "loss": 2.7475, + "theoretical_loss": 3.7123073027177407, + "tokens_seen": 837550080 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768806419257773, + "loss": 2.8197, + "theoretical_loss": 3.712278747392389, + "tokens_seen": 837615616 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037687061183550654, + "loss": 2.3862, + "theoretical_loss": 3.712250194926671, + "tokens_seen": 837681152 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768605817452357, + "loss": 2.7735, + "theoretical_loss": 3.712221645320077, + "tokens_seen": 837746688 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768505516549649, + "loss": 2.7545, + "theoretical_loss": 3.712193098572098, + "tokens_seen": 837812224 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037684052156469413, + "loss": 2.5102, + "theoretical_loss": 3.7121645546822233, + "tokens_seen": 837877760 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037683049147442326, + "loss": 2.8537, + "theoretical_loss": 3.7121360136499435, + "tokens_seen": 837943296 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768204613841525, + "loss": 2.5985, + "theoretical_loss": 3.712107475474749, + "tokens_seen": 838008832 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003768104312938817, + "loss": 2.97, + "theoretical_loss": 3.7120789401561303, + "tokens_seen": 838074368 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037680040120361086, + "loss": 2.7995, + "theoretical_loss": 3.712050407693578, + "tokens_seen": 838139904 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037679037111334004, + "loss": 3.0605, + "theoretical_loss": 3.712021878086584, + "tokens_seen": 838205440 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767803410230692, + "loss": 2.8332, + "theoretical_loss": 3.7119933513346375, + "tokens_seen": 838270976 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767703109327984, + "loss": 2.9444, + "theoretical_loss": 3.7119648274372308, + "tokens_seen": 838336512 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037676028084252764, + "loss": 3.0157, + "theoretical_loss": 3.711936306393855, + "tokens_seen": 838402048 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037675025075225676, + "loss": 2.6819, + "theoretical_loss": 3.7119077882040017, + "tokens_seen": 838467584 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376740220661986, + "loss": 2.7764, + "theoretical_loss": 3.711879272867162, + "tokens_seen": 838533120 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767301905717151, + "loss": 2.7516, + "theoretical_loss": 3.711850760382827, + "tokens_seen": 838598656 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037672016048144436, + "loss": 3.0291, + "theoretical_loss": 3.71182225075049, + "tokens_seen": 838664192 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037671013039117354, + "loss": 2.9408, + "theoretical_loss": 3.7117937439696416, + "tokens_seen": 838729728 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003767001003009027, + "loss": 2.6546, + "theoretical_loss": 3.7117652400397736, + "tokens_seen": 838795264 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 470653, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.006452798843384, + "objective/train/theoretical_loss": 3.7117367389603793, + "objective/train/tokens_used": 859320800, + "theoretical_loss": 3.7117367389603793, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766900702106319, + "loss": 2.886, + "theoretical_loss": 3.7117367389603793, + "tokens_seen": 838860800 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766800401203611, + "loss": 2.6274, + "theoretical_loss": 3.7117082407309505, + "tokens_seen": 838926336 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037667001003009027, + "loss": 2.8262, + "theoretical_loss": 3.7116797453509793, + "tokens_seen": 838991872 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766599799398195, + "loss": 2.8575, + "theoretical_loss": 3.711651252819959, + "tokens_seen": 839057408 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037664994984954863, + "loss": 2.7635, + "theoretical_loss": 3.711622763137381, + "tokens_seen": 839122944 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037663991975927786, + "loss": 2.7222, + "theoretical_loss": 3.7115942763027396, + "tokens_seen": 839188480 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037662988966900705, + "loss": 2.79, + "theoretical_loss": 3.7115657923155267, + "tokens_seen": 839254016 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766198595787362, + "loss": 2.621, + "theoretical_loss": 3.711537311175236, + "tokens_seen": 839319552 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003766098294884654, + "loss": 2.7895, + "theoretical_loss": 3.7115088328813597, + "tokens_seen": 839385088 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765997993981946, + "loss": 2.7553, + "theoretical_loss": 3.7114803574333926, + "tokens_seen": 839450624 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037658976930792377, + "loss": 2.6999, + "theoretical_loss": 3.711451884830827, + "tokens_seen": 839516160 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376579739217653, + "loss": 2.827, + "theoretical_loss": 3.7114234150731567, + "tokens_seen": 839581696 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037656970912738213, + "loss": 2.5286, + "theoretical_loss": 3.711394948159876, + "tokens_seen": 839647232 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037655967903711137, + "loss": 2.7859, + "theoretical_loss": 3.711366484090478, + "tokens_seen": 839712768 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765496489468405, + "loss": 2.8513, + "theoretical_loss": 3.7113380228644566, + "tokens_seen": 839778304 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037653961885656973, + "loss": 2.6668, + "theoretical_loss": 3.711309564481307, + "tokens_seen": 839843840 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765295887662989, + "loss": 2.8288, + "theoretical_loss": 3.711281108940522, + "tokens_seen": 839909376 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765195586760281, + "loss": 2.907, + "theoretical_loss": 3.711252656241597, + "tokens_seen": 839974912 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003765095285857573, + "loss": 2.6672, + "theoretical_loss": 3.711224206384026, + "tokens_seen": 840040448 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037649949849548645, + "loss": 2.5705, + "theoretical_loss": 3.711195759367304, + "tokens_seen": 840105984 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037648946840521564, + "loss": 2.5694, + "theoretical_loss": 3.7111673151909255, + "tokens_seen": 840171520 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037647943831494487, + "loss": 2.9223, + "theoretical_loss": 3.7111388738543853, + "tokens_seen": 840237056 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376469408224674, + "loss": 2.7859, + "theoretical_loss": 3.7111104353571784, + "tokens_seen": 840302592 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037645937813440323, + "loss": 2.8041, + "theoretical_loss": 3.7110819996988003, + "tokens_seen": 840368128 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764493480441324, + "loss": 2.6228, + "theoretical_loss": 3.711053566878746, + "tokens_seen": 840433664 + }, + { + "epoch": 0.25, + "objective/train/docs_used": 471790, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.994243860244751, + "objective/train/theoretical_loss": 3.711025136896511, + "objective/train/tokens_used": 860959200, + "theoretical_loss": 3.711025136896511, + "tokens_seen": 840499200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764393179538616, + "loss": 2.864, + "theoretical_loss": 3.711025136896511, + "tokens_seen": 840499200 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003764292878635908, + "loss": 2.5993, + "theoretical_loss": 3.7109967097515906, + "tokens_seen": 840564736 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037641925777331996, + "loss": 2.894, + "theoretical_loss": 3.7109682854434802, + "tokens_seen": 840630272 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037640922768304914, + "loss": 2.7505, + "theoretical_loss": 3.710939863971676, + "tokens_seen": 840695808 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763991975927784, + "loss": 2.8284, + "theoretical_loss": 3.7109114453356744, + "tokens_seen": 840761344 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763891675025075, + "loss": 2.9597, + "theoretical_loss": 3.710883029534971, + "tokens_seen": 840826880 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037637913741223674, + "loss": 2.5393, + "theoretical_loss": 3.710854616569061, + "tokens_seen": 840892416 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037636910732196586, + "loss": 2.6253, + "theoretical_loss": 3.710826206437442, + "tokens_seen": 840957952 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763590772316951, + "loss": 2.9871, + "theoretical_loss": 3.7107977991396104, + "tokens_seen": 841023488 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763490471414243, + "loss": 2.5868, + "theoretical_loss": 3.7107693946750624, + "tokens_seen": 841089024 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037633901705115346, + "loss": 2.8583, + "theoretical_loss": 3.710740993043294, + "tokens_seen": 841154560 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037632898696088264, + "loss": 2.642, + "theoretical_loss": 3.7107125942438035, + "tokens_seen": 841220096 + }, + { + "epoch": 0.25, + "learning_rate": 0.0003763189568706119, + "loss": 3.005, + "theoretical_loss": 3.710684198276086, + "tokens_seen": 841285632 + }, + { + "epoch": 0.25, + "learning_rate": 0.000376308926780341, + "loss": 2.6299, + "theoretical_loss": 3.7106558051396403, + "tokens_seen": 841351168 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037629889669007024, + "loss": 2.8208, + "theoretical_loss": 3.7106274148339624, + "tokens_seen": 841416704 + }, + { + "epoch": 0.25, + "learning_rate": 0.00037628886659979937, + "loss": 2.6842, + "theoretical_loss": 3.7105990273585503, + "tokens_seen": 841482240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762788365095286, + "loss": 2.8769, + "theoretical_loss": 3.7105706427129013, + "tokens_seen": 841547776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762688064192578, + "loss": 2.8247, + "theoretical_loss": 3.710542260896513, + "tokens_seen": 841613312 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037625877632898696, + "loss": 2.8089, + "theoretical_loss": 3.710513881908883, + "tokens_seen": 841678848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037624874623871615, + "loss": 2.6378, + "theoretical_loss": 3.710485505749509, + "tokens_seen": 841744384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762387161484453, + "loss": 2.6468, + "theoretical_loss": 3.7104571324178894, + "tokens_seen": 841809920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003762286860581745, + "loss": 2.7883, + "theoretical_loss": 3.710428761913522, + "tokens_seen": 841875456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037621865596790374, + "loss": 2.6884, + "theoretical_loss": 3.710400394235905, + "tokens_seen": 841940992 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037620862587763287, + "loss": 2.7462, + "theoretical_loss": 3.710372029384537, + "tokens_seen": 842006528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761985957873621, + "loss": 2.7431, + "theoretical_loss": 3.7103436673589156, + "tokens_seen": 842072064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 472419, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.929213762283325, + "objective/train/theoretical_loss": 3.710315308158541, + "objective/train/tokens_used": 862597600, + "theoretical_loss": 3.710315308158541, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037618856569709123, + "loss": 2.922, + "theoretical_loss": 3.710315308158541, + "tokens_seen": 842137600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037617853560682047, + "loss": 2.7697, + "theoretical_loss": 3.7102869517829102, + "tokens_seen": 842203136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037616850551654965, + "loss": 2.5601, + "theoretical_loss": 3.7102585982315235, + "tokens_seen": 842268672 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037615847542627883, + "loss": 2.746, + "theoretical_loss": 3.7102302475038793, + "tokens_seen": 842334208 + }, + { + "epoch": 0.26, + "learning_rate": 0.000376148445336008, + "loss": 2.8692, + "theoretical_loss": 3.7102018995994763, + "tokens_seen": 842399744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037613841524573725, + "loss": 2.585, + "theoretical_loss": 3.710173554517815, + "tokens_seen": 842465280 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037612838515546637, + "loss": 2.4941, + "theoretical_loss": 3.710145212258393, + "tokens_seen": 842530816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761183550651956, + "loss": 2.7491, + "theoretical_loss": 3.710116872820711, + "tokens_seen": 842596352 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003761083249749248, + "loss": 2.7298, + "theoretical_loss": 3.7100885362042684, + "tokens_seen": 842661888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037609829488465397, + "loss": 3.0135, + "theoretical_loss": 3.710060202408565, + "tokens_seen": 842727424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760882647943832, + "loss": 2.7064, + "theoretical_loss": 3.7100318714331006, + "tokens_seen": 842792960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037607823470411233, + "loss": 2.8233, + "theoretical_loss": 3.710003543277376, + "tokens_seen": 842858496 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037606820461384157, + "loss": 2.6252, + "theoretical_loss": 3.7099752179408894, + "tokens_seen": 842924032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760581745235707, + "loss": 2.6879, + "theoretical_loss": 3.709946895423143, + "tokens_seen": 842989568 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037604814443329993, + "loss": 2.7986, + "theoretical_loss": 3.709918575723636, + "tokens_seen": 843055104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760381143430291, + "loss": 2.8934, + "theoretical_loss": 3.7098902588418694, + "tokens_seen": 843120640 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760280842527583, + "loss": 2.7243, + "theoretical_loss": 3.7098619447773444, + "tokens_seen": 843186176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003760180541624875, + "loss": 2.7749, + "theoretical_loss": 3.7098336335295605, + "tokens_seen": 843251712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037600802407221665, + "loss": 2.7942, + "theoretical_loss": 3.70980532509802, + "tokens_seen": 843317248 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037599799398194584, + "loss": 2.636, + "theoretical_loss": 3.7097770194822224, + "tokens_seen": 843382784 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037598796389167507, + "loss": 2.5375, + "theoretical_loss": 3.70974871668167, + "tokens_seen": 843448320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759779338014042, + "loss": 2.8475, + "theoretical_loss": 3.709720416695864, + "tokens_seen": 843513856 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037596790371113343, + "loss": 2.6591, + "theoretical_loss": 3.709692119524305, + "tokens_seen": 843579392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759578736208626, + "loss": 2.4604, + "theoretical_loss": 3.709663825166496, + "tokens_seen": 843644928 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759478435305918, + "loss": 2.8007, + "theoretical_loss": 3.7096355336219373, + "tokens_seen": 843710464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 473620, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.1842246055603027, + "objective/train/theoretical_loss": 3.709607244890131, + "objective/train/tokens_used": 864236000, + "theoretical_loss": 3.709607244890131, + "tokens_seen": 843776000 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375937813440321, + "loss": 2.5554, + "theoretical_loss": 3.709607244890131, + "tokens_seen": 843776000 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037592778335005016, + "loss": 2.854, + "theoretical_loss": 3.70957895897058, + "tokens_seen": 843841536 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037591775325977934, + "loss": 2.844, + "theoretical_loss": 3.7095506758627845, + "tokens_seen": 843907072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003759077231695086, + "loss": 2.656, + "theoretical_loss": 3.7095223955662484, + "tokens_seen": 843972608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758976930792377, + "loss": 2.69, + "theoretical_loss": 3.709494118080473, + "tokens_seen": 844038144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037588766298896694, + "loss": 2.6778, + "theoretical_loss": 3.7094658434049617, + "tokens_seen": 844103680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037587763289869606, + "loss": 2.9431, + "theoretical_loss": 3.709437571539216, + "tokens_seen": 844169216 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758676028084253, + "loss": 2.8346, + "theoretical_loss": 3.709409302482739, + "tokens_seen": 844234752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758575727181545, + "loss": 2.7605, + "theoretical_loss": 3.7093810362350332, + "tokens_seen": 844300288 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037584754262788366, + "loss": 2.5804, + "theoretical_loss": 3.7093527727956026, + "tokens_seen": 844365824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037583751253761284, + "loss": 2.5707, + "theoretical_loss": 3.709324512163949, + "tokens_seen": 844431360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758274824473421, + "loss": 2.9522, + "theoretical_loss": 3.709296254339576, + "tokens_seen": 844496896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003758174523570712, + "loss": 2.804, + "theoretical_loss": 3.7092679993219875, + "tokens_seen": 844562432 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037580742226680044, + "loss": 2.6918, + "theoretical_loss": 3.709239747110686, + "tokens_seen": 844627968 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037579739217652957, + "loss": 2.7089, + "theoretical_loss": 3.7092114977051756, + "tokens_seen": 844693504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757873620862588, + "loss": 2.8012, + "theoretical_loss": 3.7091832511049603, + "tokens_seen": 844759040 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375777331995988, + "loss": 2.5886, + "theoretical_loss": 3.709155007309543, + "tokens_seen": 844824576 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037576730190571716, + "loss": 2.7937, + "theoretical_loss": 3.7091267663184286, + "tokens_seen": 844890112 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037575727181544635, + "loss": 2.6963, + "theoretical_loss": 3.7090985281311206, + "tokens_seen": 844955648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757472417251755, + "loss": 2.7533, + "theoretical_loss": 3.7090702927471235, + "tokens_seen": 845021184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757372116349047, + "loss": 2.7116, + "theoretical_loss": 3.7090420601659417, + "tokens_seen": 845086720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037572718154463394, + "loss": 2.975, + "theoretical_loss": 3.7090138303870788, + "tokens_seen": 845152256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037571715145436307, + "loss": 2.8198, + "theoretical_loss": 3.7089856034100404, + "tokens_seen": 845217792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003757071213640923, + "loss": 2.7096, + "theoretical_loss": 3.708957379234331, + "tokens_seen": 845283328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037569709127382143, + "loss": 2.7556, + "theoretical_loss": 3.7089291578594548, + "tokens_seen": 845348864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 474273, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4147276878356934, + "objective/train/theoretical_loss": 3.7089009392849173, + "objective/train/tokens_used": 865874400, + "theoretical_loss": 3.7089009392849173, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037568706118355067, + "loss": 2.8156, + "theoretical_loss": 3.7089009392849173, + "tokens_seen": 845414400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037567703109327985, + "loss": 2.8181, + "theoretical_loss": 3.7088727235102237, + "tokens_seen": 845479936 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037566700100300903, + "loss": 2.7168, + "theoretical_loss": 3.7088445105348793, + "tokens_seen": 845545472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756569709127382, + "loss": 2.7358, + "theoretical_loss": 3.7088163003583885, + "tokens_seen": 845611008 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037564694082246745, + "loss": 2.6648, + "theoretical_loss": 3.7087880929802575, + "tokens_seen": 845676544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756369107321966, + "loss": 2.7797, + "theoretical_loss": 3.708759888399992, + "tokens_seen": 845742080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003756268806419258, + "loss": 2.7489, + "theoretical_loss": 3.708731686617097, + "tokens_seen": 845807616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037561685055165494, + "loss": 2.6411, + "theoretical_loss": 3.7087034876310794, + "tokens_seen": 845873152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037560682046138417, + "loss": 2.5334, + "theoretical_loss": 3.7086752914414443, + "tokens_seen": 845938688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037559679037111335, + "loss": 2.5147, + "theoretical_loss": 3.708647098047698, + "tokens_seen": 846004224 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037558676028084253, + "loss": 2.771, + "theoretical_loss": 3.708618907449347, + "tokens_seen": 846069760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755767301905717, + "loss": 2.8255, + "theoretical_loss": 3.7085907196458976, + "tokens_seen": 846135296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755667001003009, + "loss": 2.8221, + "theoretical_loss": 3.7085625346368554, + "tokens_seen": 846200832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755566700100301, + "loss": 2.6886, + "theoretical_loss": 3.7085343524217285, + "tokens_seen": 846266368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755466399197593, + "loss": 2.5909, + "theoretical_loss": 3.708506173000022, + "tokens_seen": 846331904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037553660982948844, + "loss": 2.6621, + "theoretical_loss": 3.7084779963712435, + "tokens_seen": 846397440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755265797392177, + "loss": 2.4955, + "theoretical_loss": 3.7084498225349005, + "tokens_seen": 846462976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003755165496489468, + "loss": 2.7909, + "theoretical_loss": 3.7084216514904993, + "tokens_seen": 846528512 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037550651955867604, + "loss": 2.7867, + "theoretical_loss": 3.7083934832375474, + "tokens_seen": 846594048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754964894684052, + "loss": 2.63, + "theoretical_loss": 3.7083653177755522, + "tokens_seen": 846659584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754864593781344, + "loss": 2.647, + "theoretical_loss": 3.708337155104021, + "tokens_seen": 846725120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754764292878636, + "loss": 2.5783, + "theoretical_loss": 3.7083089952224615, + "tokens_seen": 846790656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754663991975928, + "loss": 2.7086, + "theoretical_loss": 3.7082808381303813, + "tokens_seen": 846856192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037545636910732194, + "loss": 2.8197, + "theoretical_loss": 3.7082526838272885, + "tokens_seen": 846921728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754463390170512, + "loss": 2.6207, + "theoretical_loss": 3.7082245323126903, + "tokens_seen": 846987264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 475572, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3531529903411865, + "objective/train/theoretical_loss": 3.7081963835860963, + "objective/train/tokens_used": 867512800, + "theoretical_loss": 3.7081963835860963, + "tokens_seen": 847052800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754363089267803, + "loss": 2.3995, + "theoretical_loss": 3.7081963835860963, + "tokens_seen": 847052800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037542627883650954, + "loss": 2.7983, + "theoretical_loss": 3.7081682376470133, + "tokens_seen": 847118336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754162487462387, + "loss": 2.8877, + "theoretical_loss": 3.7081400944949503, + "tokens_seen": 847183872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003754062186559679, + "loss": 2.7212, + "theoretical_loss": 3.708111954129415, + "tokens_seen": 847249408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753961885656971, + "loss": 2.747, + "theoretical_loss": 3.7080838165499177, + "tokens_seen": 847314944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037538615847542626, + "loss": 2.8743, + "theoretical_loss": 3.7080556817559653, + "tokens_seen": 847380480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037537612838515544, + "loss": 2.7958, + "theoretical_loss": 3.7080275497470674, + "tokens_seen": 847446016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753660982948847, + "loss": 2.5247, + "theoretical_loss": 3.707999420522733, + "tokens_seen": 847511552 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037535606820461386, + "loss": 2.8099, + "theoretical_loss": 3.707971294082471, + "tokens_seen": 847577088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037534603811434304, + "loss": 2.8774, + "theoretical_loss": 3.7079431704257906, + "tokens_seen": 847642624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753360080240723, + "loss": 2.8221, + "theoretical_loss": 3.7079150495522013, + "tokens_seen": 847708160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003753259779338014, + "loss": 2.7504, + "theoretical_loss": 3.7078869314612124, + "tokens_seen": 847773696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037531594784353064, + "loss": 2.5578, + "theoretical_loss": 3.707858816152334, + "tokens_seen": 847839232 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037530591775325977, + "loss": 2.8172, + "theoretical_loss": 3.707830703625075, + "tokens_seen": 847904768 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375295887662989, + "loss": 2.7628, + "theoretical_loss": 3.7078025938789456, + "tokens_seen": 847970304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752858575727182, + "loss": 2.4287, + "theoretical_loss": 3.707774486913456, + "tokens_seen": 848035840 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037527582748244736, + "loss": 2.8295, + "theoretical_loss": 3.707746382728116, + "tokens_seen": 848101376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037526579739217655, + "loss": 2.6956, + "theoretical_loss": 3.7077182813224354, + "tokens_seen": 848166912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752557673019057, + "loss": 2.5746, + "theoretical_loss": 3.7076901826959254, + "tokens_seen": 848232448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752457372116349, + "loss": 2.4742, + "theoretical_loss": 3.7076620868480963, + "tokens_seen": 848297984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037523570712136414, + "loss": 2.6048, + "theoretical_loss": 3.707633993778458, + "tokens_seen": 848363520 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037522567703109327, + "loss": 2.8559, + "theoretical_loss": 3.707605903486522, + "tokens_seen": 848429056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003752156469408225, + "loss": 2.4246, + "theoretical_loss": 3.7075778159717983, + "tokens_seen": 848494592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037520561685055163, + "loss": 2.7434, + "theoretical_loss": 3.7075497312337986, + "tokens_seen": 848560128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037519558676028087, + "loss": 2.717, + "theoretical_loss": 3.7075216492720338, + "tokens_seen": 848625664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 476007, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1800570487976074, + "objective/train/theoretical_loss": 3.7074935700860143, + "objective/train/tokens_used": 869151200, + "theoretical_loss": 3.7074935700860143, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037518555667001005, + "loss": 2.9501, + "theoretical_loss": 3.7074935700860143, + "tokens_seen": 848691200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037517552657973923, + "loss": 2.5482, + "theoretical_loss": 3.7074654936752527, + "tokens_seen": 848756736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751654964894684, + "loss": 2.6684, + "theoretical_loss": 3.70743742003926, + "tokens_seen": 848822272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037515546639919765, + "loss": 2.66, + "theoretical_loss": 3.7074093491775475, + "tokens_seen": 848887808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003751454363089268, + "loss": 2.5614, + "theoretical_loss": 3.7073812810896265, + "tokens_seen": 848953344 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375135406218656, + "loss": 2.7405, + "theoretical_loss": 3.7073532157750098, + "tokens_seen": 849018880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037512537612838514, + "loss": 2.7963, + "theoretical_loss": 3.7073251532332088, + "tokens_seen": 849084416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037511534603811437, + "loss": 2.7726, + "theoretical_loss": 3.707297093463735, + "tokens_seen": 849149952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037510531594784355, + "loss": 2.526, + "theoretical_loss": 3.7072690364661023, + "tokens_seen": 849215488 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037509528585757273, + "loss": 2.7597, + "theoretical_loss": 3.707240982239821, + "tokens_seen": 849281024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750852557673019, + "loss": 2.6971, + "theoretical_loss": 3.7072129307844053, + "tokens_seen": 849346560 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750752256770311, + "loss": 2.8605, + "theoretical_loss": 3.707184882099366, + "tokens_seen": 849412096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750651955867603, + "loss": 2.5104, + "theoretical_loss": 3.7071568361842173, + "tokens_seen": 849477632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750551654964895, + "loss": 2.5936, + "theoretical_loss": 3.7071287930384713, + "tokens_seen": 849543168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037504513540621864, + "loss": 2.5493, + "theoretical_loss": 3.7071007526616406, + "tokens_seen": 849608704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750351053159479, + "loss": 2.6672, + "theoretical_loss": 3.707072715053239, + "tokens_seen": 849674240 + }, + { + "epoch": 0.26, + "learning_rate": 0.000375025075225677, + "loss": 2.8005, + "theoretical_loss": 3.707044680212779, + "tokens_seen": 849739776 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037501504513540624, + "loss": 2.7663, + "theoretical_loss": 3.707016648139775, + "tokens_seen": 849805312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003750050150451354, + "loss": 2.8586, + "theoretical_loss": 3.7069886188337393, + "tokens_seen": 849870848 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749949849548646, + "loss": 2.801, + "theoretical_loss": 3.7069605922941857, + "tokens_seen": 849936384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749849548645938, + "loss": 2.865, + "theoretical_loss": 3.706932568520628, + "tokens_seen": 850001920 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374974924774323, + "loss": 2.7011, + "theoretical_loss": 3.70690454751258, + "tokens_seen": 850067456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037496489468405214, + "loss": 2.7869, + "theoretical_loss": 3.7068765292695556, + "tokens_seen": 850132992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749548645937814, + "loss": 2.889, + "theoretical_loss": 3.7068485137910683, + "tokens_seen": 850198528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749448345035105, + "loss": 2.6505, + "theoretical_loss": 3.7068205010766335, + "tokens_seen": 850264064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 477422, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5388436317443848, + "objective/train/theoretical_loss": 3.7067924911257646, + "objective/train/tokens_used": 870789600, + "theoretical_loss": 3.7067924911257646, + "tokens_seen": 850329600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037493480441323974, + "loss": 2.5794, + "theoretical_loss": 3.7067924911257646, + "tokens_seen": 850329600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749247743229689, + "loss": 2.5744, + "theoretical_loss": 3.7067644839379756, + "tokens_seen": 850395136 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749147442326981, + "loss": 2.7938, + "theoretical_loss": 3.706736479512782, + "tokens_seen": 850460672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003749047141424273, + "loss": 2.7906, + "theoretical_loss": 3.7067084778496984, + "tokens_seen": 850526208 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037489468405215646, + "loss": 2.7479, + "theoretical_loss": 3.7066804789482384, + "tokens_seen": 850591744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037488465396188564, + "loss": 2.5192, + "theoretical_loss": 3.706652482807918, + "tokens_seen": 850657280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748746238716149, + "loss": 2.5515, + "theoretical_loss": 3.7066244894282523, + "tokens_seen": 850722816 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374864593781344, + "loss": 2.8724, + "theoretical_loss": 3.7065964988087554, + "tokens_seen": 850788352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037485456369107324, + "loss": 2.6995, + "theoretical_loss": 3.7065685109489435, + "tokens_seen": 850853888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037484453360080237, + "loss": 2.5975, + "theoretical_loss": 3.7065405258483315, + "tokens_seen": 850919424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748345035105316, + "loss": 2.9188, + "theoretical_loss": 3.7065125435064354, + "tokens_seen": 850984960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003748244734202608, + "loss": 2.7731, + "theoretical_loss": 3.7064845639227704, + "tokens_seen": 851050496 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037481444332998997, + "loss": 2.7823, + "theoretical_loss": 3.706456587096852, + "tokens_seen": 851116032 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037480441323971915, + "loss": 2.5889, + "theoretical_loss": 3.706428613028197, + "tokens_seen": 851181568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747943831494484, + "loss": 2.7635, + "theoretical_loss": 3.706400641716321, + "tokens_seen": 851247104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747843530591775, + "loss": 2.6144, + "theoretical_loss": 3.7063726731607396, + "tokens_seen": 851312640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037477432296890675, + "loss": 2.663, + "theoretical_loss": 3.7063447073609694, + "tokens_seen": 851378176 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037476429287863587, + "loss": 2.6883, + "theoretical_loss": 3.706316744316527, + "tokens_seen": 851443712 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747542627883651, + "loss": 2.6976, + "theoretical_loss": 3.7062887840269276, + "tokens_seen": 851509248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003747442326980943, + "loss": 2.778, + "theoretical_loss": 3.7062608264916896, + "tokens_seen": 851574784 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037473420260782347, + "loss": 2.7051, + "theoretical_loss": 3.7062328717103292, + "tokens_seen": 851640320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037472417251755265, + "loss": 2.8306, + "theoretical_loss": 3.7062049196823628, + "tokens_seen": 851705856 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037471414242728183, + "loss": 2.4378, + "theoretical_loss": 3.7061769704073075, + "tokens_seen": 851771392 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374704112337011, + "loss": 2.5944, + "theoretical_loss": 3.706149023884681, + "tokens_seen": 851836928 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037469408224674025, + "loss": 2.783, + "theoretical_loss": 3.7061210801139994, + "tokens_seen": 851902464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 478724, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.294400453567505, + "objective/train/theoretical_loss": 3.706093139094781, + "objective/train/tokens_used": 872428000, + "theoretical_loss": 3.706093139094781, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746840521564694, + "loss": 2.5393, + "theoretical_loss": 3.706093139094781, + "tokens_seen": 851968000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003746740220661986, + "loss": 2.7337, + "theoretical_loss": 3.706065200826543, + "tokens_seen": 852033536 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037466399197592774, + "loss": 2.5102, + "theoretical_loss": 3.7060372653088027, + "tokens_seen": 852099072 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374653961885657, + "loss": 2.5369, + "theoretical_loss": 3.706009332541078, + "tokens_seen": 852164608 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037464393179538615, + "loss": 2.7483, + "theoretical_loss": 3.7059814025228865, + "tokens_seen": 852230144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037463390170511534, + "loss": 2.7406, + "theoretical_loss": 3.7059534752537466, + "tokens_seen": 852295680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037462387161484457, + "loss": 2.7608, + "theoretical_loss": 3.705925550733176, + "tokens_seen": 852361216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037461384152457375, + "loss": 2.7804, + "theoretical_loss": 3.7058976289606935, + "tokens_seen": 852426752 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037460381143430293, + "loss": 2.7932, + "theoretical_loss": 3.7058697099358167, + "tokens_seen": 852492288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745937813440321, + "loss": 2.7504, + "theoretical_loss": 3.705841793658064, + "tokens_seen": 852557824 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745837512537613, + "loss": 2.8511, + "theoretical_loss": 3.7058138801269545, + "tokens_seen": 852623360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745737211634905, + "loss": 2.6997, + "theoretical_loss": 3.7057859693420063, + "tokens_seen": 852688896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745636910732197, + "loss": 2.8532, + "theoretical_loss": 3.7057580613027383, + "tokens_seen": 852754432 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037455366098294884, + "loss": 2.5094, + "theoretical_loss": 3.70573015600867, + "tokens_seen": 852819968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745436308926781, + "loss": 2.8309, + "theoretical_loss": 3.70570225345932, + "tokens_seen": 852885504 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745336008024072, + "loss": 2.7093, + "theoretical_loss": 3.7056743536542074, + "tokens_seen": 852951040 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037452357071213644, + "loss": 2.8218, + "theoretical_loss": 3.7056464565928517, + "tokens_seen": 853016576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745135406218656, + "loss": 2.5999, + "theoretical_loss": 3.705618562274772, + "tokens_seen": 853082112 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003745035105315948, + "loss": 2.562, + "theoretical_loss": 3.7055906706994883, + "tokens_seen": 853147648 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374493480441324, + "loss": 2.7136, + "theoretical_loss": 3.7055627818665196, + "tokens_seen": 853213184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744834503510532, + "loss": 2.7561, + "theoretical_loss": 3.7055348957753864, + "tokens_seen": 853278720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037447342026078234, + "loss": 2.7161, + "theoretical_loss": 3.705507012425608, + "tokens_seen": 853344256 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744633901705116, + "loss": 2.7196, + "theoretical_loss": 3.7054791318167046, + "tokens_seen": 853409792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744533600802407, + "loss": 2.8581, + "theoretical_loss": 3.7054512539481967, + "tokens_seen": 853475328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037444332998996994, + "loss": 2.7128, + "theoretical_loss": 3.705423378819604, + "tokens_seen": 853540864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 479323, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.415107488632202, + "objective/train/theoretical_loss": 3.7053955064304467, + "objective/train/tokens_used": 874066400, + "theoretical_loss": 3.7053955064304467, + "tokens_seen": 853606400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744332998996991, + "loss": 2.7135, + "theoretical_loss": 3.7053955064304467, + "tokens_seen": 853606400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744232698094283, + "loss": 2.5923, + "theoretical_loss": 3.705367636780246, + "tokens_seen": 853671936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003744132397191575, + "loss": 2.6997, + "theoretical_loss": 3.705339769868522, + "tokens_seen": 853737472 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037440320962888666, + "loss": 2.777, + "theoretical_loss": 3.7053119056947956, + "tokens_seen": 853803008 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037439317953861585, + "loss": 2.5342, + "theoretical_loss": 3.7052840442585877, + "tokens_seen": 853868544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743831494483451, + "loss": 2.7932, + "theoretical_loss": 3.7052561855594193, + "tokens_seen": 853934080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743731193580742, + "loss": 2.5486, + "theoretical_loss": 3.705228329596811, + "tokens_seen": 853999616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037436308926780344, + "loss": 2.8331, + "theoretical_loss": 3.705200476370285, + "tokens_seen": 854065152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037435305917753257, + "loss": 2.6312, + "theoretical_loss": 3.7051726258793622, + "tokens_seen": 854130688 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743430290872618, + "loss": 2.6545, + "theoretical_loss": 3.7051447781235636, + "tokens_seen": 854196224 + }, + { + "epoch": 0.26, + "learning_rate": 0.000374332998996991, + "loss": 2.6804, + "theoretical_loss": 3.7051169331024107, + "tokens_seen": 854261760 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037432296890672017, + "loss": 2.646, + "theoretical_loss": 3.7050890908154264, + "tokens_seen": 854327296 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037431293881644935, + "loss": 2.5474, + "theoretical_loss": 3.7050612512621313, + "tokens_seen": 854392832 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003743029087261786, + "loss": 2.6389, + "theoretical_loss": 3.7050334144420476, + "tokens_seen": 854458368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742928786359077, + "loss": 2.6941, + "theoretical_loss": 3.7050055803546975, + "tokens_seen": 854523904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037428284854563695, + "loss": 2.6424, + "theoretical_loss": 3.7049777489996036, + "tokens_seen": 854589440 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037427281845536607, + "loss": 2.6623, + "theoretical_loss": 3.7049499203762872, + "tokens_seen": 854654976 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742627883650953, + "loss": 2.6297, + "theoretical_loss": 3.704922094484272, + "tokens_seen": 854720512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742527582748245, + "loss": 2.774, + "theoretical_loss": 3.7048942713230795, + "tokens_seen": 854786048 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037424272818455367, + "loss": 2.73, + "theoretical_loss": 3.7048664508922324, + "tokens_seen": 854851584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037423269809428285, + "loss": 2.7748, + "theoretical_loss": 3.704838633191254, + "tokens_seen": 854917120 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037422266800401203, + "loss": 2.6596, + "theoretical_loss": 3.7048108182196673, + "tokens_seen": 854982656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003742126379137412, + "loss": 2.7619, + "theoretical_loss": 3.704783005976995, + "tokens_seen": 855048192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037420260782347045, + "loss": 2.7137, + "theoretical_loss": 3.7047551964627594, + "tokens_seen": 855113728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741925777331996, + "loss": 2.8362, + "theoretical_loss": 3.704727389676486, + "tokens_seen": 855179264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 479893, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.8994100093841553, + "objective/train/theoretical_loss": 3.7046995856176954, + "objective/train/tokens_used": 875704800, + "theoretical_loss": 3.7046995856176954, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741825476429288, + "loss": 2.3327, + "theoretical_loss": 3.7046995856176954, + "tokens_seen": 855244800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037417251755265794, + "loss": 2.7426, + "theoretical_loss": 3.704671784285913, + "tokens_seen": 855310336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741624874623872, + "loss": 2.7, + "theoretical_loss": 3.704643985680662, + "tokens_seen": 855375872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037415245737211635, + "loss": 2.7171, + "theoretical_loss": 3.704616189801466, + "tokens_seen": 855441408 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037414242728184554, + "loss": 2.7728, + "theoretical_loss": 3.704588396647848, + "tokens_seen": 855506944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741323971915747, + "loss": 2.7179, + "theoretical_loss": 3.704560606219334, + "tokens_seen": 855572480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037412236710130395, + "loss": 2.4159, + "theoretical_loss": 3.7045328185154465, + "tokens_seen": 855638016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741123370110331, + "loss": 2.6816, + "theoretical_loss": 3.7045050335357104, + "tokens_seen": 855703552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003741023069207623, + "loss": 2.7457, + "theoretical_loss": 3.7044772512796493, + "tokens_seen": 855769088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037409227683049144, + "loss": 2.4778, + "theoretical_loss": 3.7044494717467886, + "tokens_seen": 855834624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740822467402207, + "loss": 2.759, + "theoretical_loss": 3.704421694936652, + "tokens_seen": 855900160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037407221664994986, + "loss": 2.7427, + "theoretical_loss": 3.704393920848765, + "tokens_seen": 855965696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037406218655967904, + "loss": 2.8548, + "theoretical_loss": 3.704366149482652, + "tokens_seen": 856031232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740521564694082, + "loss": 2.6829, + "theoretical_loss": 3.7043383808378376, + "tokens_seen": 856096768 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740421263791374, + "loss": 2.4935, + "theoretical_loss": 3.7043106149138474, + "tokens_seen": 856162304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740320962888666, + "loss": 2.4234, + "theoretical_loss": 3.7042828517102064, + "tokens_seen": 856227840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740220661985958, + "loss": 2.6602, + "theoretical_loss": 3.7042550912264405, + "tokens_seen": 856293376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037401203610832494, + "loss": 2.6378, + "theoretical_loss": 3.7042273334620734, + "tokens_seen": 856358912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003740020060180542, + "loss": 2.5173, + "theoretical_loss": 3.7041995784166324, + "tokens_seen": 856424448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037399197592778336, + "loss": 2.8294, + "theoretical_loss": 3.7041718260896426, + "tokens_seen": 856489984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037398194583751254, + "loss": 2.7361, + "theoretical_loss": 3.7041440764806293, + "tokens_seen": 856555520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739719157472417, + "loss": 2.9064, + "theoretical_loss": 3.7041163295891186, + "tokens_seen": 856621056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739618856569709, + "loss": 2.7333, + "theoretical_loss": 3.7040885854146364, + "tokens_seen": 856686592 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739518555667001, + "loss": 2.7232, + "theoretical_loss": 3.7040608439567095, + "tokens_seen": 856752128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739418254764293, + "loss": 2.6206, + "theoretical_loss": 3.7040331052148634, + "tokens_seen": 856817664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 480416, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0554239749908447, + "objective/train/theoretical_loss": 3.704005369188625, + "objective/train/tokens_used": 877343200, + "theoretical_loss": 3.704005369188625, + "tokens_seen": 856883200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037393179538615845, + "loss": 2.8683, + "theoretical_loss": 3.704005369188625, + "tokens_seen": 856883200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739217652958877, + "loss": 2.8753, + "theoretical_loss": 3.70397763587752, + "tokens_seen": 856948736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003739117352056168, + "loss": 2.4591, + "theoretical_loss": 3.7039499052810765, + "tokens_seen": 857014272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037390170511534605, + "loss": 2.7519, + "theoretical_loss": 3.703922177398819, + "tokens_seen": 857079808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738916750250752, + "loss": 2.7931, + "theoretical_loss": 3.7038944522302764, + "tokens_seen": 857145344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738816449348044, + "loss": 2.6563, + "theoretical_loss": 3.7038667297749743, + "tokens_seen": 857210880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037387161484453364, + "loss": 2.6392, + "theoretical_loss": 3.703839010032441, + "tokens_seen": 857276416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037386158475426277, + "loss": 2.6964, + "theoretical_loss": 3.703811293002202, + "tokens_seen": 857341952 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373851554663992, + "loss": 2.5881, + "theoretical_loss": 3.7037835786837867, + "tokens_seen": 857407488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738415245737212, + "loss": 2.697, + "theoretical_loss": 3.7037558670767208, + "tokens_seen": 857473024 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037383149448345037, + "loss": 2.611, + "theoretical_loss": 3.7037281581805326, + "tokens_seen": 857538560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037382146439317955, + "loss": 2.5995, + "theoretical_loss": 3.7037004519947496, + "tokens_seen": 857604096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738114343029088, + "loss": 2.3132, + "theoretical_loss": 3.7036727485188994, + "tokens_seen": 857669632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003738014042126379, + "loss": 2.7532, + "theoretical_loss": 3.7036450477525102, + "tokens_seen": 857735168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037379137412236715, + "loss": 2.7108, + "theoretical_loss": 3.7036173496951106, + "tokens_seen": 857800704 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037378134403209627, + "loss": 2.9879, + "theoretical_loss": 3.703589654346227, + "tokens_seen": 857866240 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737713139418255, + "loss": 2.9101, + "theoretical_loss": 3.7035619617053896, + "tokens_seen": 857931776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737612838515547, + "loss": 2.6775, + "theoretical_loss": 3.703534271772125, + "tokens_seen": 857997312 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037375125376128387, + "loss": 2.6034, + "theoretical_loss": 3.7035065845459636, + "tokens_seen": 858062848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037374122367101305, + "loss": 2.6527, + "theoretical_loss": 3.7034789000264325, + "tokens_seen": 858128384 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037373119358074223, + "loss": 2.7135, + "theoretical_loss": 3.7034512182130612, + "tokens_seen": 858193920 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737211634904714, + "loss": 2.5494, + "theoretical_loss": 3.703423539105378, + "tokens_seen": 858259456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037371113340020065, + "loss": 2.7046, + "theoretical_loss": 3.703395862702912, + "tokens_seen": 858324992 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003737011033099298, + "loss": 2.5938, + "theoretical_loss": 3.703368189005193, + "tokens_seen": 858390528 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373691073219659, + "loss": 2.5671, + "theoretical_loss": 3.70334051801175, + "tokens_seen": 858456064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 481568, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5986602306365967, + "objective/train/theoretical_loss": 3.703312849722111, + "objective/train/tokens_used": 878981600, + "theoretical_loss": 3.703312849722111, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037368104312938814, + "loss": 2.8, + "theoretical_loss": 3.703312849722111, + "tokens_seen": 858521600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736710130391174, + "loss": 2.6659, + "theoretical_loss": 3.7032851841358063, + "tokens_seen": 858587136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037366098294884655, + "loss": 2.431, + "theoretical_loss": 3.7032575212523664, + "tokens_seen": 858652672 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037365095285857574, + "loss": 2.788, + "theoretical_loss": 3.7032298610713195, + "tokens_seen": 858718208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736409227683049, + "loss": 2.5209, + "theoretical_loss": 3.7032022035921965, + "tokens_seen": 858783744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037363089267803415, + "loss": 2.6643, + "theoretical_loss": 3.7031745488145265, + "tokens_seen": 858849280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736208625877633, + "loss": 2.8994, + "theoretical_loss": 3.70314689673784, + "tokens_seen": 858914816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003736108324974925, + "loss": 2.9014, + "theoretical_loss": 3.703119247361667, + "tokens_seen": 858980352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037360080240722164, + "loss": 2.8771, + "theoretical_loss": 3.7030916006855374, + "tokens_seen": 859045888 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735907723169509, + "loss": 2.8042, + "theoretical_loss": 3.7030639567089825, + "tokens_seen": 859111424 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037358074222668006, + "loss": 2.6583, + "theoretical_loss": 3.7030363154315316, + "tokens_seen": 859176960 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037357071213640924, + "loss": 2.6478, + "theoretical_loss": 3.7030086768527166, + "tokens_seen": 859242496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735606820461384, + "loss": 2.5504, + "theoretical_loss": 3.702981040972067, + "tokens_seen": 859308032 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735506519558676, + "loss": 2.7879, + "theoretical_loss": 3.7029534077891144, + "tokens_seen": 859373568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735406218655968, + "loss": 2.9783, + "theoretical_loss": 3.70292577730339, + "tokens_seen": 859439104 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373530591775326, + "loss": 2.9209, + "theoretical_loss": 3.702898149514424, + "tokens_seen": 859504640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037352056168505514, + "loss": 2.592, + "theoretical_loss": 3.702870524421748, + "tokens_seen": 859570176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003735105315947844, + "loss": 2.816, + "theoretical_loss": 3.7028429020248934, + "tokens_seen": 859635712 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037350050150451356, + "loss": 2.4548, + "theoretical_loss": 3.7028152823233924, + "tokens_seen": 859701248 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037349047141424274, + "loss": 2.8427, + "theoretical_loss": 3.702787665316775, + "tokens_seen": 859766784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734804413239719, + "loss": 2.6741, + "theoretical_loss": 3.7027600510045735, + "tokens_seen": 859832320 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734704112337011, + "loss": 2.844, + "theoretical_loss": 3.7027324393863204, + "tokens_seen": 859897856 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734603811434303, + "loss": 2.8788, + "theoretical_loss": 3.702704830461547, + "tokens_seen": 859963392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734503510531595, + "loss": 2.5995, + "theoretical_loss": 3.702677224229785, + "tokens_seen": 860028928 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037344032096288865, + "loss": 2.7775, + "theoretical_loss": 3.702649620690567, + "tokens_seen": 860094464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 482749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.050727367401123, + "objective/train/theoretical_loss": 3.7026220198434254, + "objective/train/tokens_used": 880620000, + "theoretical_loss": 3.7026220198434254, + "tokens_seen": 860160000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734302908726179, + "loss": 2.7565, + "theoretical_loss": 3.7026220198434254, + "tokens_seen": 860160000 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373420260782347, + "loss": 2.5725, + "theoretical_loss": 3.702594421687892, + "tokens_seen": 860225536 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037341023069207625, + "loss": 2.5902, + "theoretical_loss": 3.7025668262234994, + "tokens_seen": 860291072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003734002006018054, + "loss": 2.8756, + "theoretical_loss": 3.702539233449781, + "tokens_seen": 860356608 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733901705115346, + "loss": 2.4553, + "theoretical_loss": 3.702511643366268, + "tokens_seen": 860422144 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733801404212638, + "loss": 2.7468, + "theoretical_loss": 3.702484055972495, + "tokens_seen": 860487680 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037337011033099297, + "loss": 2.7126, + "theoretical_loss": 3.702456471267994, + "tokens_seen": 860553216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037336008024072215, + "loss": 2.7523, + "theoretical_loss": 3.7024288892522983, + "tokens_seen": 860618752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733500501504514, + "loss": 2.6144, + "theoretical_loss": 3.702401309924941, + "tokens_seen": 860684288 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733400200601805, + "loss": 2.5013, + "theoretical_loss": 3.702373733285455, + "tokens_seen": 860749824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037332998996990975, + "loss": 2.5324, + "theoretical_loss": 3.7023461593333744, + "tokens_seen": 860815360 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037331995987963893, + "loss": 2.6501, + "theoretical_loss": 3.7023185880682323, + "tokens_seen": 860880896 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003733099297893681, + "loss": 2.8881, + "theoretical_loss": 3.702291019489562, + "tokens_seen": 860946432 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732998996990973, + "loss": 2.7107, + "theoretical_loss": 3.7022634535968986, + "tokens_seen": 861011968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732898696088265, + "loss": 2.8248, + "theoretical_loss": 3.7022358903897747, + "tokens_seen": 861077504 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037327983951855565, + "loss": 2.6673, + "theoretical_loss": 3.702208329867725, + "tokens_seen": 861143040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732698094282849, + "loss": 2.7935, + "theoretical_loss": 3.7021807720302835, + "tokens_seen": 861208576 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373259779338014, + "loss": 2.6674, + "theoretical_loss": 3.702153216876984, + "tokens_seen": 861274112 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037324974924774325, + "loss": 2.7553, + "theoretical_loss": 3.702125664407361, + "tokens_seen": 861339648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732397191574724, + "loss": 2.7629, + "theoretical_loss": 3.7020981146209495, + "tokens_seen": 861405184 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732296890672016, + "loss": 2.6363, + "theoretical_loss": 3.7020705675172834, + "tokens_seen": 861470720 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003732196589769308, + "loss": 2.9832, + "theoretical_loss": 3.702043023095898, + "tokens_seen": 861536256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037320962888666, + "loss": 2.7581, + "theoretical_loss": 3.7020154813563275, + "tokens_seen": 861601792 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037319959879638916, + "loss": 2.6392, + "theoretical_loss": 3.7019879422981075, + "tokens_seen": 861667328 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037318956870611834, + "loss": 2.7417, + "theoretical_loss": 3.7019604059207727, + "tokens_seen": 861732864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 483414, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.466357707977295, + "objective/train/theoretical_loss": 3.701932872223858, + "objective/train/tokens_used": 882258400, + "theoretical_loss": 3.701932872223858, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731795386158475, + "loss": 2.7092, + "theoretical_loss": 3.701932872223858, + "tokens_seen": 861798400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037316950852557675, + "loss": 2.5774, + "theoretical_loss": 3.701905341206899, + "tokens_seen": 861863936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731594784353059, + "loss": 2.5797, + "theoretical_loss": 3.701877812869431, + "tokens_seen": 861929472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731494483450351, + "loss": 2.4721, + "theoretical_loss": 3.70185028721099, + "tokens_seen": 861995008 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731394182547643, + "loss": 2.6781, + "theoretical_loss": 3.7018227642311112, + "tokens_seen": 862060544 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731293881644935, + "loss": 2.6883, + "theoretical_loss": 3.70179524392933, + "tokens_seen": 862126080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003731193580742227, + "loss": 2.7031, + "theoretical_loss": 3.7017677263051825, + "tokens_seen": 862191616 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037310932798395184, + "loss": 2.7407, + "theoretical_loss": 3.701740211358205, + "tokens_seen": 862257152 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730992978936811, + "loss": 2.8633, + "theoretical_loss": 3.7017126990879334, + "tokens_seen": 862322688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037308926780341026, + "loss": 2.856, + "theoretical_loss": 3.7016851894939036, + "tokens_seen": 862388224 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037307923771313944, + "loss": 2.7041, + "theoretical_loss": 3.701657682575653, + "tokens_seen": 862453760 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730692076228686, + "loss": 2.8978, + "theoretical_loss": 3.7016301783327163, + "tokens_seen": 862519296 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730591775325978, + "loss": 2.5655, + "theoretical_loss": 3.7016026767646313, + "tokens_seen": 862584832 + }, + { + "epoch": 0.26, + "learning_rate": 0.000373049147442327, + "loss": 2.4688, + "theoretical_loss": 3.7015751778709345, + "tokens_seen": 862650368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730391173520562, + "loss": 2.8267, + "theoretical_loss": 3.701547681651162, + "tokens_seen": 862715904 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037302908726178534, + "loss": 2.7697, + "theoretical_loss": 3.701520188104852, + "tokens_seen": 862781440 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003730190571715146, + "loss": 2.7066, + "theoretical_loss": 3.70149269723154, + "tokens_seen": 862846976 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037300902708124376, + "loss": 2.8466, + "theoretical_loss": 3.7014652090307645, + "tokens_seen": 862912512 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037299899699097294, + "loss": 2.5227, + "theoretical_loss": 3.701437723502062, + "tokens_seen": 862978048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729889669007021, + "loss": 2.7204, + "theoretical_loss": 3.70141024064497, + "tokens_seen": 863043584 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729789368104313, + "loss": 2.5038, + "theoretical_loss": 3.7013827604590253, + "tokens_seen": 863109120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729689067201605, + "loss": 2.7124, + "theoretical_loss": 3.7013552829437675, + "tokens_seen": 863174656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729588766298897, + "loss": 2.8171, + "theoretical_loss": 3.701327808098732, + "tokens_seen": 863240192 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037294884653961885, + "loss": 2.9852, + "theoretical_loss": 3.7013003359234578, + "tokens_seen": 863305728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729388164493481, + "loss": 2.6671, + "theoretical_loss": 3.701272866417482, + "tokens_seen": 863371264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 484154, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.915800094604492, + "objective/train/theoretical_loss": 3.701245399580344, + "objective/train/tokens_used": 883896800, + "theoretical_loss": 3.701245399580344, + "tokens_seen": 863436800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729287863590772, + "loss": 2.7206, + "theoretical_loss": 3.701245399580344, + "tokens_seen": 863436800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037291875626880645, + "loss": 2.7457, + "theoretical_loss": 3.7012179354115813, + "tokens_seen": 863502336 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003729087261785356, + "loss": 2.8516, + "theoretical_loss": 3.701190473910732, + "tokens_seen": 863567872 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728986960882648, + "loss": 2.6364, + "theoretical_loss": 3.7011630150773343, + "tokens_seen": 863633408 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372888665997994, + "loss": 2.7432, + "theoretical_loss": 3.7011355589109276, + "tokens_seen": 863698944 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037287863590772317, + "loss": 2.6735, + "theoretical_loss": 3.7011081054110493, + "tokens_seen": 863764480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037286860581745235, + "loss": 2.675, + "theoretical_loss": 3.7010806545772397, + "tokens_seen": 863830016 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728585757271816, + "loss": 2.8821, + "theoretical_loss": 3.7010532064090356, + "tokens_seen": 863895552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728485456369107, + "loss": 2.5917, + "theoretical_loss": 3.701025760905978, + "tokens_seen": 863961088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037283851554663995, + "loss": 2.761, + "theoretical_loss": 3.7009983180676045, + "tokens_seen": 864026624 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037282848545636913, + "loss": 2.8932, + "theoretical_loss": 3.7009708778934556, + "tokens_seen": 864092160 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728184553660983, + "loss": 2.8616, + "theoretical_loss": 3.7009434403830697, + "tokens_seen": 864157696 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003728084252758275, + "loss": 2.7249, + "theoretical_loss": 3.700916005535986, + "tokens_seen": 864223232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727983951855567, + "loss": 2.6967, + "theoretical_loss": 3.7008885733517447, + "tokens_seen": 864288768 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037278836509528585, + "loss": 2.6364, + "theoretical_loss": 3.7008611438298855, + "tokens_seen": 864354304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727783350050151, + "loss": 2.8586, + "theoretical_loss": 3.7008337169699477, + "tokens_seen": 864419840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727683049147442, + "loss": 2.7462, + "theoretical_loss": 3.700806292771472, + "tokens_seen": 864485376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037275827482447345, + "loss": 2.6446, + "theoretical_loss": 3.7007788712339966, + "tokens_seen": 864550912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727482447342026, + "loss": 2.58, + "theoretical_loss": 3.700751452357064, + "tokens_seen": 864616448 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727382146439318, + "loss": 2.6979, + "theoretical_loss": 3.7007240361402127, + "tokens_seen": 864681984 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372728184553661, + "loss": 2.7701, + "theoretical_loss": 3.700696622582984, + "tokens_seen": 864747520 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003727181544633902, + "loss": 2.7526, + "theoretical_loss": 3.7006692116849176, + "tokens_seen": 864813056 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037270812437311936, + "loss": 2.7688, + "theoretical_loss": 3.7006418034455546, + "tokens_seen": 864878592 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037269809428284854, + "loss": 2.9445, + "theoretical_loss": 3.7006143978644355, + "tokens_seen": 864944128 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726880641925777, + "loss": 2.7108, + "theoretical_loss": 3.7005869949411014, + "tokens_seen": 865009664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 484565, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.951662063598633, + "objective/train/theoretical_loss": 3.7005595946750924, + "objective/train/tokens_used": 885535200, + "theoretical_loss": 3.7005595946750924, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037267803410230696, + "loss": 2.7365, + "theoretical_loss": 3.7005595946750924, + "tokens_seen": 865075200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726680040120361, + "loss": 2.5957, + "theoretical_loss": 3.70053219706595, + "tokens_seen": 865140736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726579739217653, + "loss": 2.8784, + "theoretical_loss": 3.7005048021132163, + "tokens_seen": 865206272 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726479438314945, + "loss": 2.6853, + "theoretical_loss": 3.700477409816431, + "tokens_seen": 865271808 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726379137412237, + "loss": 3.0309, + "theoretical_loss": 3.7004500201751362, + "tokens_seen": 865337344 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037262788365095286, + "loss": 2.9844, + "theoretical_loss": 3.7004226331888734, + "tokens_seen": 865402880 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037261785356068204, + "loss": 2.457, + "theoretical_loss": 3.700395248857184, + "tokens_seen": 865468416 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003726078234704112, + "loss": 2.9597, + "theoretical_loss": 3.7003678671796103, + "tokens_seen": 865533952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037259779338014046, + "loss": 2.7733, + "theoretical_loss": 3.7003404881556934, + "tokens_seen": 865599488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725877632898696, + "loss": 2.6146, + "theoretical_loss": 3.7003131117849755, + "tokens_seen": 865665024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725777331995988, + "loss": 2.8385, + "theoretical_loss": 3.7002857380669987, + "tokens_seen": 865730560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037256770310932795, + "loss": 2.5251, + "theoretical_loss": 3.7002583670013047, + "tokens_seen": 865796096 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725576730190572, + "loss": 2.7264, + "theoretical_loss": 3.700230998587437, + "tokens_seen": 865861632 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037254764292878636, + "loss": 2.718, + "theoretical_loss": 3.7002036328249366, + "tokens_seen": 865927168 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037253761283851554, + "loss": 2.9428, + "theoretical_loss": 3.7001762697133467, + "tokens_seen": 865992704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725275827482447, + "loss": 2.7134, + "theoretical_loss": 3.70014890925221, + "tokens_seen": 866058240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037251755265797396, + "loss": 2.866, + "theoretical_loss": 3.700121551441069, + "tokens_seen": 866123776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003725075225677031, + "loss": 2.5316, + "theoretical_loss": 3.7000941962794673, + "tokens_seen": 866189312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724974924774323, + "loss": 2.955, + "theoretical_loss": 3.7000668437669466, + "tokens_seen": 866254848 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037248746238716145, + "loss": 2.7666, + "theoretical_loss": 3.7000394939030503, + "tokens_seen": 866320384 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724774322968907, + "loss": 2.803, + "theoretical_loss": 3.7000121466873224, + "tokens_seen": 866385920 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037246740220661987, + "loss": 2.553, + "theoretical_loss": 3.6999848021193054, + "tokens_seen": 866451456 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037245737211634905, + "loss": 2.7715, + "theoretical_loss": 3.6999574601985428, + "tokens_seen": 866516992 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037244734202607823, + "loss": 2.645, + "theoretical_loss": 3.699930120924579, + "tokens_seen": 866582528 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724373119358074, + "loss": 2.8014, + "theoretical_loss": 3.699902784296956, + "tokens_seen": 866648064 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 485783, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.643681049346924, + "objective/train/theoretical_loss": 3.6998754503152194, + "objective/train/tokens_used": 887173600, + "theoretical_loss": 3.6998754503152194, + "tokens_seen": 866713600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724272818455366, + "loss": 2.8837, + "theoretical_loss": 3.6998754503152194, + "tokens_seen": 866713600 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003724172517552658, + "loss": 2.6677, + "theoretical_loss": 3.6998481189789114, + "tokens_seen": 866779136 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037240722166499495, + "loss": 2.7465, + "theoretical_loss": 3.6998207902875775, + "tokens_seen": 866844672 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723971915747242, + "loss": 2.8192, + "theoretical_loss": 3.6997934642407606, + "tokens_seen": 866910208 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723871614844533, + "loss": 2.8605, + "theoretical_loss": 3.6997661408380056, + "tokens_seen": 866975744 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037237713139418255, + "loss": 2.7537, + "theoretical_loss": 3.6997388200788563, + "tokens_seen": 867041280 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723671013039118, + "loss": 2.7422, + "theoretical_loss": 3.6997115019628577, + "tokens_seen": 867106816 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723570712136409, + "loss": 2.7609, + "theoretical_loss": 3.699684186489554, + "tokens_seen": 867172352 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037234704112337015, + "loss": 2.7326, + "theoretical_loss": 3.6996568736584896, + "tokens_seen": 867237888 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037233701103309933, + "loss": 2.6956, + "theoretical_loss": 3.69962956346921, + "tokens_seen": 867303424 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723269809428285, + "loss": 2.6659, + "theoretical_loss": 3.69960225592126, + "tokens_seen": 867368960 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723169508525577, + "loss": 2.8015, + "theoretical_loss": 3.699574951014183, + "tokens_seen": 867434496 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003723069207622869, + "loss": 2.6047, + "theoretical_loss": 3.6995476487475267, + "tokens_seen": 867500032 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037229689067201605, + "loss": 2.6313, + "theoretical_loss": 3.6995203491208346, + "tokens_seen": 867565568 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722868605817453, + "loss": 2.7724, + "theoretical_loss": 3.6994930521336524, + "tokens_seen": 867631104 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722768304914744, + "loss": 2.7818, + "theoretical_loss": 3.699465757785526, + "tokens_seen": 867696640 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037226680040120365, + "loss": 2.6688, + "theoretical_loss": 3.699438466076, + "tokens_seen": 867762176 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722567703109328, + "loss": 2.6619, + "theoretical_loss": 3.699411177004621, + "tokens_seen": 867827712 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372246740220662, + "loss": 2.7528, + "theoretical_loss": 3.6993838905709344, + "tokens_seen": 867893248 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722367101303912, + "loss": 2.6117, + "theoretical_loss": 3.6993566067744856, + "tokens_seen": 867958784 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003722266800401204, + "loss": 2.5744, + "theoretical_loss": 3.699329325614822, + "tokens_seen": 868024320 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037221664994984956, + "loss": 2.6486, + "theoretical_loss": 3.6993020470914884, + "tokens_seen": 868089856 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037220661985957874, + "loss": 2.8173, + "theoretical_loss": 3.6992747712040317, + "tokens_seen": 868155392 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721965897693079, + "loss": 2.6862, + "theoretical_loss": 3.699247497951998, + "tokens_seen": 868220928 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037218655967903716, + "loss": 2.7351, + "theoretical_loss": 3.699220227334934, + "tokens_seen": 868286464 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 486350, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.766869068145752, + "objective/train/theoretical_loss": 3.699192959352386, + "objective/train/tokens_used": 888812000, + "theoretical_loss": 3.699192959352386, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721765295887663, + "loss": 2.5503, + "theoretical_loss": 3.699192959352386, + "tokens_seen": 868352000 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721664994984955, + "loss": 3.0311, + "theoretical_loss": 3.6991656940039004, + "tokens_seen": 868417536 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721564694082247, + "loss": 2.6552, + "theoretical_loss": 3.6991384312890245, + "tokens_seen": 868483072 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721464393179539, + "loss": 2.4007, + "theoretical_loss": 3.6991111712073055, + "tokens_seen": 868548608 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037213640922768306, + "loss": 2.809, + "theoretical_loss": 3.69908391375829, + "tokens_seen": 868614144 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037212637913741224, + "loss": 2.7626, + "theoretical_loss": 3.699056658941525, + "tokens_seen": 868679680 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003721163490471414, + "loss": 2.519, + "theoretical_loss": 3.6990294067565577, + "tokens_seen": 868745216 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037210631895687066, + "loss": 2.9509, + "theoretical_loss": 3.699002157202936, + "tokens_seen": 868810752 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720962888665998, + "loss": 2.7953, + "theoretical_loss": 3.6989749102802065, + "tokens_seen": 868876288 + }, + { + "epoch": 0.26, + "learning_rate": 0.000372086258776329, + "loss": 2.6473, + "theoretical_loss": 3.6989476659879177, + "tokens_seen": 868941824 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037207622868605815, + "loss": 2.6791, + "theoretical_loss": 3.6989204243256166, + "tokens_seen": 869007360 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720661985957874, + "loss": 2.9017, + "theoretical_loss": 3.698893185292852, + "tokens_seen": 869072896 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037205616850551656, + "loss": 2.7375, + "theoretical_loss": 3.6988659488891704, + "tokens_seen": 869138432 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037204613841524575, + "loss": 3.008, + "theoretical_loss": 3.698838715114121, + "tokens_seen": 869203968 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720361083249749, + "loss": 2.8997, + "theoretical_loss": 3.6988114839672512, + "tokens_seen": 869269504 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037202607823470416, + "loss": 2.8079, + "theoretical_loss": 3.6987842554481096, + "tokens_seen": 869335040 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720160481444333, + "loss": 2.7345, + "theoretical_loss": 3.6987570295562446, + "tokens_seen": 869400576 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003720060180541625, + "loss": 2.7938, + "theoretical_loss": 3.6987298062912046, + "tokens_seen": 869466112 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037199598796389165, + "loss": 2.7915, + "theoretical_loss": 3.6987025856525384, + "tokens_seen": 869531648 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719859578736209, + "loss": 2.8572, + "theoretical_loss": 3.6986753676397948, + "tokens_seen": 869597184 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037197592778335007, + "loss": 2.687, + "theoretical_loss": 3.698648152252521, + "tokens_seen": 869662720 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037196589769307925, + "loss": 2.7066, + "theoretical_loss": 3.6986209394902687, + "tokens_seen": 869728256 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037195586760280843, + "loss": 2.6519, + "theoretical_loss": 3.6985937293525843, + "tokens_seen": 869793792 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719458375125376, + "loss": 2.6958, + "theoretical_loss": 3.6985665218390187, + "tokens_seen": 869859328 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719358074222668, + "loss": 2.6778, + "theoretical_loss": 3.6985393169491205, + "tokens_seen": 869924864 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 487478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4885756969451904, + "objective/train/theoretical_loss": 3.698512114682439, + "objective/train/tokens_used": 890450400, + "theoretical_loss": 3.698512114682439, + "tokens_seen": 869990400 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371925777331996, + "loss": 2.6768, + "theoretical_loss": 3.698512114682439, + "tokens_seen": 869990400 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037191574724172515, + "loss": 2.6099, + "theoretical_loss": 3.698484915038524, + "tokens_seen": 870055936 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003719057171514544, + "loss": 2.6162, + "theoretical_loss": 3.698457718016925, + "tokens_seen": 870121472 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718956870611835, + "loss": 3.0378, + "theoretical_loss": 3.6984305236171915, + "tokens_seen": 870187008 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037188565697091275, + "loss": 2.4252, + "theoretical_loss": 3.698403331838873, + "tokens_seen": 870252544 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037187562688064193, + "loss": 2.5673, + "theoretical_loss": 3.6983761426815205, + "tokens_seen": 870318080 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718655967903711, + "loss": 2.865, + "theoretical_loss": 3.6983489561446827, + "tokens_seen": 870383616 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718555667001003, + "loss": 2.7507, + "theoretical_loss": 3.698321772227911, + "tokens_seen": 870449152 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037184553660982953, + "loss": 2.5034, + "theoretical_loss": 3.6982945909307547, + "tokens_seen": 870514688 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037183550651955866, + "loss": 2.5666, + "theoretical_loss": 3.698267412252765, + "tokens_seen": 870580224 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003718254764292879, + "loss": 2.7432, + "theoretical_loss": 3.6982402361934916, + "tokens_seen": 870645760 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371815446339017, + "loss": 2.6505, + "theoretical_loss": 3.698213062752486, + "tokens_seen": 870711296 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037180541624874625, + "loss": 2.8117, + "theoretical_loss": 3.698185891929297, + "tokens_seen": 870776832 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037179538615847544, + "loss": 2.7201, + "theoretical_loss": 3.698158723723478, + "tokens_seen": 870842368 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717853560682046, + "loss": 2.5757, + "theoretical_loss": 3.6981315581345786, + "tokens_seen": 870907904 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717753259779338, + "loss": 2.8679, + "theoretical_loss": 3.698104395162149, + "tokens_seen": 870973440 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371765295887663, + "loss": 2.5399, + "theoretical_loss": 3.6980772348057425, + "tokens_seen": 871038976 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037175526579739216, + "loss": 2.7885, + "theoretical_loss": 3.6980500770649085, + "tokens_seen": 871104512 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717452357071214, + "loss": 2.707, + "theoretical_loss": 3.6980229219391987, + "tokens_seen": 871170048 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717352056168505, + "loss": 2.6914, + "theoretical_loss": 3.6979957694281653, + "tokens_seen": 871235584 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037172517552657976, + "loss": 2.7263, + "theoretical_loss": 3.697968619531359, + "tokens_seen": 871301120 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717151454363089, + "loss": 2.5678, + "theoretical_loss": 3.697941472248332, + "tokens_seen": 871366656 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003717051153460381, + "loss": 2.5981, + "theoretical_loss": 3.6979143275786366, + "tokens_seen": 871432192 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716950852557673, + "loss": 2.7973, + "theoretical_loss": 3.697887185521824, + "tokens_seen": 871497728 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716850551654965, + "loss": 2.897, + "theoretical_loss": 3.6978600460774462, + "tokens_seen": 871563264 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 487872, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4374098777770996, + "objective/train/theoretical_loss": 3.6978329092450557, + "objective/train/tokens_used": 892088800, + "theoretical_loss": 3.6978329092450557, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037167502507522566, + "loss": 2.6415, + "theoretical_loss": 3.6978329092450557, + "tokens_seen": 871628800 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716649949849549, + "loss": 2.8264, + "theoretical_loss": 3.6978057750242046, + "tokens_seen": 871694336 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371654964894684, + "loss": 2.8088, + "theoretical_loss": 3.697778643414445, + "tokens_seen": 871759872 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037164493480441326, + "loss": 2.6045, + "theoretical_loss": 3.69775151441533, + "tokens_seen": 871825408 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716349047141424, + "loss": 2.806, + "theoretical_loss": 3.697724388026412, + "tokens_seen": 871890944 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003716248746238716, + "loss": 2.804, + "theoretical_loss": 3.6976972642472425, + "tokens_seen": 871956480 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037161484453360086, + "loss": 2.7078, + "theoretical_loss": 3.697670143077376, + "tokens_seen": 872022016 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037160481444333, + "loss": 2.6623, + "theoretical_loss": 3.697643024516365, + "tokens_seen": 872087552 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715947843530592, + "loss": 2.8471, + "theoretical_loss": 3.697615908563762, + "tokens_seen": 872153088 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037158475426278835, + "loss": 2.5388, + "theoretical_loss": 3.6975887952191204, + "tokens_seen": 872218624 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715747241725176, + "loss": 2.7085, + "theoretical_loss": 3.697561684481993, + "tokens_seen": 872284160 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037156469408224676, + "loss": 2.8703, + "theoretical_loss": 3.6975345763519343, + "tokens_seen": 872349696 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037155466399197595, + "loss": 2.6017, + "theoretical_loss": 3.697507470828497, + "tokens_seen": 872415232 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715446339017051, + "loss": 2.5009, + "theoretical_loss": 3.6974803679112345, + "tokens_seen": 872480768 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037153460381143436, + "loss": 2.5021, + "theoretical_loss": 3.697453267599701, + "tokens_seen": 872546304 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715245737211635, + "loss": 3.0077, + "theoretical_loss": 3.6974261698934496, + "tokens_seen": 872611840 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003715145436308927, + "loss": 2.8439, + "theoretical_loss": 3.6973990747920347, + "tokens_seen": 872677376 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037150451354062185, + "loss": 2.7113, + "theoretical_loss": 3.69737198229501, + "tokens_seen": 872742912 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714944834503511, + "loss": 2.5746, + "theoretical_loss": 3.6973448924019308, + "tokens_seen": 872808448 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037148445336008027, + "loss": 2.7075, + "theoretical_loss": 3.6973178051123496, + "tokens_seen": 872873984 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037147442326980945, + "loss": 2.5569, + "theoretical_loss": 3.697290720425822, + "tokens_seen": 872939520 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037146439317953863, + "loss": 2.6775, + "theoretical_loss": 3.6972636383419015, + "tokens_seen": 873005056 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714543630892678, + "loss": 2.7166, + "theoretical_loss": 3.6972365588601432, + "tokens_seen": 873070592 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371444332998997, + "loss": 2.6978, + "theoretical_loss": 3.697209481980102, + "tokens_seen": 873136128 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037143430290872623, + "loss": 2.7577, + "theoretical_loss": 3.697182407701332, + "tokens_seen": 873201664 + }, + { + "epoch": 0.26, + "objective/train/docs_used": 488898, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9824957847595215, + "objective/train/theoretical_loss": 3.697155336023389, + "objective/train/tokens_used": 893727200, + "theoretical_loss": 3.697155336023389, + "tokens_seen": 873267200 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037142427281845535, + "loss": 2.7698, + "theoretical_loss": 3.697155336023389, + "tokens_seen": 873267200 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714142427281846, + "loss": 2.9909, + "theoretical_loss": 3.6971282669458274, + "tokens_seen": 873332736 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003714042126379137, + "loss": 2.651, + "theoretical_loss": 3.6971012004682025, + "tokens_seen": 873398272 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037139418254764295, + "loss": 2.7109, + "theoretical_loss": 3.697074136590069, + "tokens_seen": 873463808 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037138415245737213, + "loss": 2.8332, + "theoretical_loss": 3.6970470753109828, + "tokens_seen": 873529344 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713741223671013, + "loss": 2.7787, + "theoretical_loss": 3.6970200166304994, + "tokens_seen": 873594880 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713640922768305, + "loss": 2.9767, + "theoretical_loss": 3.696992960548174, + "tokens_seen": 873660416 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037135406218655973, + "loss": 2.8765, + "theoretical_loss": 3.6969659070635625, + "tokens_seen": 873725952 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037134403209628886, + "loss": 2.7108, + "theoretical_loss": 3.696938856176221, + "tokens_seen": 873791488 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713340020060181, + "loss": 2.7555, + "theoretical_loss": 3.6969118078857046, + "tokens_seen": 873857024 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003713239719157472, + "loss": 2.7623, + "theoretical_loss": 3.6968847621915693, + "tokens_seen": 873922560 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037131394182547645, + "loss": 2.7329, + "theoretical_loss": 3.696857719093372, + "tokens_seen": 873988096 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037130391173520564, + "loss": 2.6343, + "theoretical_loss": 3.6968306785906684, + "tokens_seen": 874053632 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712938816449348, + "loss": 2.5664, + "theoretical_loss": 3.6968036406830147, + "tokens_seen": 874119168 + }, + { + "epoch": 0.26, + "learning_rate": 0.000371283851554664, + "loss": 2.7419, + "theoretical_loss": 3.6967766053699673, + "tokens_seen": 874184704 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712738214643932, + "loss": 2.6855, + "theoretical_loss": 3.6967495726510835, + "tokens_seen": 874250240 + }, + { + "epoch": 0.26, + "learning_rate": 0.00037126379137412236, + "loss": 2.8358, + "theoretical_loss": 3.6967225425259187, + "tokens_seen": 874315776 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712537612838516, + "loss": 2.7927, + "theoretical_loss": 3.6966955149940306, + "tokens_seen": 874381312 + }, + { + "epoch": 0.26, + "learning_rate": 0.0003712437311935807, + "loss": 2.6235, + "theoretical_loss": 3.696668490054976, + "tokens_seen": 874446848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037123370110330996, + "loss": 2.5521, + "theoretical_loss": 3.6966414677083113, + "tokens_seen": 874512384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712236710130391, + "loss": 2.936, + "theoretical_loss": 3.696614447953594, + "tokens_seen": 874577920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712136409227683, + "loss": 2.5652, + "theoretical_loss": 3.696587430790381, + "tokens_seen": 874643456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003712036108324975, + "loss": 2.7471, + "theoretical_loss": 3.6965604162182304, + "tokens_seen": 874708992 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711935807422267, + "loss": 2.6699, + "theoretical_loss": 3.696533404236699, + "tokens_seen": 874774528 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037118355065195586, + "loss": 2.8556, + "theoretical_loss": 3.6965063948453434, + "tokens_seen": 874840064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 489972, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.883964776992798, + "objective/train/theoretical_loss": 3.6964793880437226, + "objective/train/tokens_used": 895365600, + "theoretical_loss": 3.6964793880437226, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711735205616851, + "loss": 2.8779, + "theoretical_loss": 3.6964793880437226, + "tokens_seen": 874905600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711634904714142, + "loss": 2.854, + "theoretical_loss": 3.696452383831394, + "tokens_seen": 874971136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037115346038114346, + "loss": 2.901, + "theoretical_loss": 3.6964253822079147, + "tokens_seen": 875036672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711434302908726, + "loss": 3.0085, + "theoretical_loss": 3.696398383172844, + "tokens_seen": 875102208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711334002006018, + "loss": 2.5525, + "theoretical_loss": 3.696371386725739, + "tokens_seen": 875167744 + }, + { + "epoch": 0.27, + "learning_rate": 0.000371123370110331, + "loss": 2.8206, + "theoretical_loss": 3.696344392866158, + "tokens_seen": 875233280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003711133400200602, + "loss": 2.9085, + "theoretical_loss": 3.6963174015936593, + "tokens_seen": 875298816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037110330992978937, + "loss": 2.7336, + "theoretical_loss": 3.6962904129078016, + "tokens_seen": 875364352 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037109327983951855, + "loss": 2.5856, + "theoretical_loss": 3.6962634268081427, + "tokens_seen": 875429888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037108324974924773, + "loss": 2.5817, + "theoretical_loss": 3.6962364432942416, + "tokens_seen": 875495424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037107321965897696, + "loss": 2.808, + "theoretical_loss": 3.6962094623656574, + "tokens_seen": 875560960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710631895687061, + "loss": 2.8817, + "theoretical_loss": 3.696182484021948, + "tokens_seen": 875626496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710531594784353, + "loss": 2.7093, + "theoretical_loss": 3.696155508262673, + "tokens_seen": 875692032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037104312938816445, + "loss": 2.671, + "theoretical_loss": 3.6961285350873916, + "tokens_seen": 875757568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003710330992978937, + "loss": 2.6938, + "theoretical_loss": 3.6961015644956623, + "tokens_seen": 875823104 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037102306920762287, + "loss": 2.7216, + "theoretical_loss": 3.696074596487045, + "tokens_seen": 875888640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037101303911735205, + "loss": 2.5135, + "theoretical_loss": 3.6960476310610986, + "tokens_seen": 875954176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037100300902708123, + "loss": 2.663, + "theoretical_loss": 3.696020668217382, + "tokens_seen": 876019712 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037099297893681047, + "loss": 2.7203, + "theoretical_loss": 3.6959937079554566, + "tokens_seen": 876085248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709829488465396, + "loss": 2.7627, + "theoretical_loss": 3.69596675027488, + "tokens_seen": 876150784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037097291875626883, + "loss": 2.8865, + "theoretical_loss": 3.6959397951752133, + "tokens_seen": 876216320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037096288866599796, + "loss": 2.6588, + "theoretical_loss": 3.695912842656016, + "tokens_seen": 876281856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709528585757272, + "loss": 2.8049, + "theoretical_loss": 3.695885892716848, + "tokens_seen": 876347392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709428284854564, + "loss": 2.7291, + "theoretical_loss": 3.6958589453572697, + "tokens_seen": 876412928 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037093279839518555, + "loss": 2.826, + "theoretical_loss": 3.695832000576841, + "tokens_seen": 876478464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 490667, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.061556816101074, + "objective/train/theoretical_loss": 3.6958050583751225, + "objective/train/tokens_used": 897004000, + "theoretical_loss": 3.6958050583751225, + "tokens_seen": 876544000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037092276830491474, + "loss": 2.8808, + "theoretical_loss": 3.6958050583751225, + "tokens_seen": 876544000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709127382146439, + "loss": 2.7035, + "theoretical_loss": 3.6957781187516745, + "tokens_seen": 876609536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003709027081243731, + "loss": 2.8415, + "theoretical_loss": 3.6957511817060578, + "tokens_seen": 876675072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037089267803410233, + "loss": 2.6895, + "theoretical_loss": 3.695724247237832, + "tokens_seen": 876740608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037088264794383146, + "loss": 2.6108, + "theoretical_loss": 3.6956973153465595, + "tokens_seen": 876806144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708726178535607, + "loss": 2.8491, + "theoretical_loss": 3.6956703860318, + "tokens_seen": 876871680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037086258776328993, + "loss": 2.8227, + "theoretical_loss": 3.695643459293115, + "tokens_seen": 876937216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037085255767301906, + "loss": 2.6601, + "theoretical_loss": 3.695616535130065, + "tokens_seen": 877002752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708425275827483, + "loss": 2.7217, + "theoretical_loss": 3.6955896135422117, + "tokens_seen": 877068288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003708324974924774, + "loss": 2.8116, + "theoretical_loss": 3.6955626945291167, + "tokens_seen": 877133824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037082246740220665, + "loss": 2.6787, + "theoretical_loss": 3.6955357780903406, + "tokens_seen": 877199360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037081243731193584, + "loss": 2.5873, + "theoretical_loss": 3.6955088642254452, + "tokens_seen": 877264896 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370802407221665, + "loss": 2.8596, + "theoretical_loss": 3.6954819529339926, + "tokens_seen": 877330432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707923771313942, + "loss": 2.6279, + "theoretical_loss": 3.695455044215544, + "tokens_seen": 877395968 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707823470411234, + "loss": 2.6978, + "theoretical_loss": 3.695428138069661, + "tokens_seen": 877461504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037077231695085256, + "loss": 2.6799, + "theoretical_loss": 3.6954012344959066, + "tokens_seen": 877527040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707622868605818, + "loss": 2.7125, + "theoretical_loss": 3.6953743334938416, + "tokens_seen": 877592576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707522567703109, + "loss": 2.7077, + "theoretical_loss": 3.6953474350630287, + "tokens_seen": 877658112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037074222668004016, + "loss": 2.81, + "theoretical_loss": 3.6953205392030304, + "tokens_seen": 877723648 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707321965897693, + "loss": 2.6282, + "theoretical_loss": 3.6952936459134085, + "tokens_seen": 877789184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707221664994985, + "loss": 2.9963, + "theoretical_loss": 3.6952667551937264, + "tokens_seen": 877854720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707121364092277, + "loss": 2.6417, + "theoretical_loss": 3.695239867043546, + "tokens_seen": 877920256 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003707021063189569, + "loss": 2.889, + "theoretical_loss": 3.6952129814624297, + "tokens_seen": 877985792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037069207622868606, + "loss": 2.7111, + "theoretical_loss": 3.695186098449941, + "tokens_seen": 878051328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706820461384153, + "loss": 2.7687, + "theoretical_loss": 3.6951592180056423, + "tokens_seen": 878116864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 491863, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.594583511352539, + "objective/train/theoretical_loss": 3.6951323401290974, + "objective/train/tokens_used": 898642400, + "theoretical_loss": 3.6951323401290974, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706720160481444, + "loss": 2.6835, + "theoretical_loss": 3.6951323401290974, + "tokens_seen": 878182400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037066198595787366, + "loss": 2.8534, + "theoretical_loss": 3.695105464819868, + "tokens_seen": 878247936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706519558676028, + "loss": 2.758, + "theoretical_loss": 3.6950785920775187, + "tokens_seen": 878313472 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370641925777332, + "loss": 2.9426, + "theoretical_loss": 3.6950517219016117, + "tokens_seen": 878379008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706318956870612, + "loss": 2.8975, + "theoretical_loss": 3.695024854291712, + "tokens_seen": 878444544 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003706218655967904, + "loss": 2.549, + "theoretical_loss": 3.6949979892473817, + "tokens_seen": 878510080 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037061183550651957, + "loss": 2.9225, + "theoretical_loss": 3.694971126768185, + "tokens_seen": 878575616 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037060180541624875, + "loss": 2.5897, + "theoretical_loss": 3.6949442668536854, + "tokens_seen": 878641152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037059177532597793, + "loss": 2.8333, + "theoretical_loss": 3.694917409503447, + "tokens_seen": 878706688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037058174523570716, + "loss": 3.032, + "theoretical_loss": 3.6948905547170336, + "tokens_seen": 878772224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705717151454363, + "loss": 2.8523, + "theoretical_loss": 3.6948637024940094, + "tokens_seen": 878837760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705616850551655, + "loss": 2.8115, + "theoretical_loss": 3.694836852833939, + "tokens_seen": 878903296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037055165496489465, + "loss": 2.618, + "theoretical_loss": 3.694810005736386, + "tokens_seen": 878968832 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003705416248746239, + "loss": 2.7893, + "theoretical_loss": 3.694783161200916, + "tokens_seen": 879034368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037053159478435307, + "loss": 2.8638, + "theoretical_loss": 3.6947563192270914, + "tokens_seen": 879099904 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037052156469408225, + "loss": 2.5287, + "theoretical_loss": 3.6947294798144785, + "tokens_seen": 879165440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037051153460381143, + "loss": 2.7751, + "theoretical_loss": 3.6947026429626417, + "tokens_seen": 879230976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037050150451354067, + "loss": 2.7179, + "theoretical_loss": 3.6946758086711453, + "tokens_seen": 879296512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704914744232698, + "loss": 2.8058, + "theoretical_loss": 3.694648976939555, + "tokens_seen": 879362048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037048144433299903, + "loss": 2.7717, + "theoretical_loss": 3.694622147767435, + "tokens_seen": 879427584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037047141424272816, + "loss": 2.6822, + "theoretical_loss": 3.694595321154351, + "tokens_seen": 879493120 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704613841524574, + "loss": 2.5861, + "theoretical_loss": 3.6945684970998682, + "tokens_seen": 879558656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704513540621866, + "loss": 2.6915, + "theoretical_loss": 3.694541675603552, + "tokens_seen": 879624192 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037044132397191575, + "loss": 2.7335, + "theoretical_loss": 3.6945148566649677, + "tokens_seen": 879689728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037043129388164494, + "loss": 2.8133, + "theoretical_loss": 3.694488040283681, + "tokens_seen": 879755264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 492132, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4907939434051514, + "objective/train/theoretical_loss": 3.6944612264592576, + "objective/train/tokens_used": 900280800, + "theoretical_loss": 3.6944612264592576, + "tokens_seen": 879820800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704212637913741, + "loss": 2.7163, + "theoretical_loss": 3.6944612264592576, + "tokens_seen": 879820800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003704112337011033, + "loss": 2.9041, + "theoretical_loss": 3.694434415191263, + "tokens_seen": 879886336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037040120361083253, + "loss": 2.8063, + "theoretical_loss": 3.6944076064792633, + "tokens_seen": 879951872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037039117352056166, + "loss": 2.7982, + "theoretical_loss": 3.6943808003228247, + "tokens_seen": 880017408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703811434302909, + "loss": 2.8089, + "theoretical_loss": 3.694353996721513, + "tokens_seen": 880082944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703711133400201, + "loss": 2.5928, + "theoretical_loss": 3.694327195674894, + "tokens_seen": 880148480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037036108324974926, + "loss": 2.8074, + "theoretical_loss": 3.694300397182535, + "tokens_seen": 880214016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037035105315947844, + "loss": 2.8316, + "theoretical_loss": 3.6942736012440016, + "tokens_seen": 880279552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703410230692076, + "loss": 2.7329, + "theoretical_loss": 3.6942468078588604, + "tokens_seen": 880345088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703309929789368, + "loss": 2.9809, + "theoretical_loss": 3.6942200170266792, + "tokens_seen": 880410624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037032096288866604, + "loss": 2.8692, + "theoretical_loss": 3.694193228747023, + "tokens_seen": 880476160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037031093279839516, + "loss": 2.7912, + "theoretical_loss": 3.6941664430194594, + "tokens_seen": 880541696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003703009027081244, + "loss": 2.738, + "theoretical_loss": 3.6941396598435556, + "tokens_seen": 880607232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702908726178535, + "loss": 2.8439, + "theoretical_loss": 3.6941128792188787, + "tokens_seen": 880672768 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037028084252758276, + "loss": 2.624, + "theoretical_loss": 3.6940861011449955, + "tokens_seen": 880738304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037027081243731194, + "loss": 2.9418, + "theoretical_loss": 3.694059325621473, + "tokens_seen": 880803840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702607823470411, + "loss": 2.5756, + "theoretical_loss": 3.6940325526478794, + "tokens_seen": 880869376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702507522567703, + "loss": 2.7575, + "theoretical_loss": 3.6940057822237815, + "tokens_seen": 880934912 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702407221664995, + "loss": 2.9176, + "theoretical_loss": 3.6939790143487476, + "tokens_seen": 881000448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037023069207622867, + "loss": 2.6867, + "theoretical_loss": 3.6939522490223444, + "tokens_seen": 881065984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003702206619859579, + "loss": 2.8547, + "theoretical_loss": 3.6939254862441397, + "tokens_seen": 881131520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037021063189568703, + "loss": 2.504, + "theoretical_loss": 3.6938987260137024, + "tokens_seen": 881197056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037020060180541626, + "loss": 2.8788, + "theoretical_loss": 3.6938719683306003, + "tokens_seen": 881262592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037019057171514544, + "loss": 2.5329, + "theoretical_loss": 3.6938452131944004, + "tokens_seen": 881328128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701805416248746, + "loss": 2.8602, + "theoretical_loss": 3.6938184606046724, + "tokens_seen": 881393664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 493018, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3380496501922607, + "objective/train/theoretical_loss": 3.6937917105609834, + "objective/train/tokens_used": 901919200, + "theoretical_loss": 3.6937917105609834, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701705115346038, + "loss": 2.6497, + "theoretical_loss": 3.6937917105609834, + "tokens_seen": 881459200 + }, + { + "epoch": 0.27, + "learning_rate": 0.000370160481444333, + "loss": 2.9965, + "theoretical_loss": 3.6937649630629026, + "tokens_seen": 881524736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037015045135406217, + "loss": 2.7815, + "theoretical_loss": 3.6937382181099983, + "tokens_seen": 881590272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003701404212637914, + "loss": 2.5983, + "theoretical_loss": 3.6937114757018388, + "tokens_seen": 881655808 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037013039117352053, + "loss": 2.9515, + "theoretical_loss": 3.6936847358379934, + "tokens_seen": 881721344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037012036108324977, + "loss": 2.7363, + "theoretical_loss": 3.693657998518031, + "tokens_seen": 881786880 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037011033099297895, + "loss": 2.8001, + "theoretical_loss": 3.6936312637415196, + "tokens_seen": 881852416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037010030090270813, + "loss": 2.7913, + "theoretical_loss": 3.6936045315080293, + "tokens_seen": 881917952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037009027081243736, + "loss": 2.5676, + "theoretical_loss": 3.6935778018171286, + "tokens_seen": 881983488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700802407221665, + "loss": 2.8248, + "theoretical_loss": 3.6935510746683873, + "tokens_seen": 882049024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700702106318957, + "loss": 2.7201, + "theoretical_loss": 3.693524350061374, + "tokens_seen": 882114560 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037006018054162485, + "loss": 2.7187, + "theoretical_loss": 3.693497627995659, + "tokens_seen": 882180096 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003700501504513541, + "loss": 2.9054, + "theoretical_loss": 3.693470908470812, + "tokens_seen": 882245632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037004012036108327, + "loss": 2.4963, + "theoretical_loss": 3.6934441914864014, + "tokens_seen": 882311168 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037003009027081245, + "loss": 2.6246, + "theoretical_loss": 3.693417477041998, + "tokens_seen": 882376704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037002006018054163, + "loss": 2.7817, + "theoretical_loss": 3.693390765137172, + "tokens_seen": 882442240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037001003009027087, + "loss": 2.7273, + "theoretical_loss": 3.693364055771492, + "tokens_seen": 882507776 + }, + { + "epoch": 0.27, + "learning_rate": 0.00037, + "loss": 2.8035, + "theoretical_loss": 3.69333734894453, + "tokens_seen": 882573312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036998996990972923, + "loss": 2.9529, + "theoretical_loss": 3.693310644655855, + "tokens_seen": 882638848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036997993981945836, + "loss": 2.8309, + "theoretical_loss": 3.693283942905037, + "tokens_seen": 882704384 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699699097291876, + "loss": 3.0268, + "theoretical_loss": 3.6932572436916473, + "tokens_seen": 882769920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699598796389168, + "loss": 2.9085, + "theoretical_loss": 3.6932305470152564, + "tokens_seen": 882835456 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036994984954864595, + "loss": 2.8015, + "theoretical_loss": 3.693203852875434, + "tokens_seen": 882900992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036993981945837514, + "loss": 2.5191, + "theoretical_loss": 3.693177161271752, + "tokens_seen": 882966528 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699297893681043, + "loss": 2.8193, + "theoretical_loss": 3.6931504722037802, + "tokens_seen": 883032064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 493587, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.08097767829895, + "objective/train/theoretical_loss": 3.6931237856710903, + "objective/train/tokens_used": 903557600, + "theoretical_loss": 3.6931237856710903, + "tokens_seen": 883097600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003699197592778335, + "loss": 2.993, + "theoretical_loss": 3.6931237856710903, + "tokens_seen": 883097600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036990972918756273, + "loss": 2.6583, + "theoretical_loss": 3.693097101673253, + "tokens_seen": 883163136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036989969909729186, + "loss": 2.7407, + "theoretical_loss": 3.6930704202098394, + "tokens_seen": 883228672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698896690070211, + "loss": 2.965, + "theoretical_loss": 3.6930437412804213, + "tokens_seen": 883294208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698796389167503, + "loss": 2.8227, + "theoretical_loss": 3.693017064884569, + "tokens_seen": 883359744 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036986960882647946, + "loss": 2.8113, + "theoretical_loss": 3.692990391021855, + "tokens_seen": 883425280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036985957873620864, + "loss": 2.77, + "theoretical_loss": 3.69296371969185, + "tokens_seen": 883490816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698495486459378, + "loss": 2.7267, + "theoretical_loss": 3.692937050894127, + "tokens_seen": 883556352 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369839518555667, + "loss": 2.5705, + "theoretical_loss": 3.692910384628256, + "tokens_seen": 883621888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036982948846539624, + "loss": 2.9653, + "theoretical_loss": 3.69288372089381, + "tokens_seen": 883687424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036981945837512536, + "loss": 2.7199, + "theoretical_loss": 3.6928570596903607, + "tokens_seen": 883752960 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003698094282848546, + "loss": 2.7355, + "theoretical_loss": 3.6928304010174804, + "tokens_seen": 883818496 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697993981945837, + "loss": 2.5483, + "theoretical_loss": 3.692803744874741, + "tokens_seen": 883884032 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036978936810431296, + "loss": 2.8324, + "theoretical_loss": 3.692777091261715, + "tokens_seen": 883949568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036977933801404214, + "loss": 2.5926, + "theoretical_loss": 3.692750440177975, + "tokens_seen": 884015104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697693079237713, + "loss": 2.809, + "theoretical_loss": 3.6927237916230933, + "tokens_seen": 884080640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697592778335005, + "loss": 2.8743, + "theoretical_loss": 3.692697145596642, + "tokens_seen": 884146176 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697492477432297, + "loss": 2.6491, + "theoretical_loss": 3.6926705020981943, + "tokens_seen": 884211712 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036973921765295887, + "loss": 2.8035, + "theoretical_loss": 3.692643861127323, + "tokens_seen": 884277248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003697291875626881, + "loss": 2.9265, + "theoretical_loss": 3.692617222683601, + "tokens_seen": 884342784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036971915747241723, + "loss": 2.7762, + "theoretical_loss": 3.6925905867666016, + "tokens_seen": 884408320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036970912738214646, + "loss": 2.8945, + "theoretical_loss": 3.6925639533758976, + "tokens_seen": 884473856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036969909729187565, + "loss": 2.9008, + "theoretical_loss": 3.692537322511062, + "tokens_seen": 884539392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696890672016048, + "loss": 2.7927, + "theoretical_loss": 3.6925106941716686, + "tokens_seen": 884604928 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369679037111334, + "loss": 2.5308, + "theoretical_loss": 3.692484068357291, + "tokens_seen": 884670464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 494919, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.518890857696533, + "objective/train/theoretical_loss": 3.692457445067501, + "objective/train/tokens_used": 905196000, + "theoretical_loss": 3.692457445067501, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696690070210632, + "loss": 2.6387, + "theoretical_loss": 3.692457445067501, + "tokens_seen": 884736000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036965897693079237, + "loss": 2.8342, + "theoretical_loss": 3.692430824301875, + "tokens_seen": 884801536 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696489468405216, + "loss": 2.6288, + "theoretical_loss": 3.692404206059985, + "tokens_seen": 884867072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036963891675025073, + "loss": 2.7104, + "theoretical_loss": 3.692377590341405, + "tokens_seen": 884932608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036962888665997997, + "loss": 2.8244, + "theoretical_loss": 3.6923509771457095, + "tokens_seen": 884998144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003696188565697091, + "loss": 2.8814, + "theoretical_loss": 3.692324366472472, + "tokens_seen": 885063680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036960882647943833, + "loss": 2.9496, + "theoretical_loss": 3.692297758321267, + "tokens_seen": 885129216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695987963891675, + "loss": 2.5732, + "theoretical_loss": 3.692271152691669, + "tokens_seen": 885194752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695887662988967, + "loss": 2.6226, + "theoretical_loss": 3.6922445495832514, + "tokens_seen": 885260288 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036957873620862587, + "loss": 2.8412, + "theoretical_loss": 3.69221794899559, + "tokens_seen": 885325824 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036956870611835505, + "loss": 2.6286, + "theoretical_loss": 3.6921913509282582, + "tokens_seen": 885391360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036955867602808424, + "loss": 2.7941, + "theoretical_loss": 3.692164755380831, + "tokens_seen": 885456896 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036954864593781347, + "loss": 2.7921, + "theoretical_loss": 3.692138162352884, + "tokens_seen": 885522432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695386158475426, + "loss": 2.793, + "theoretical_loss": 3.6921115718439905, + "tokens_seen": 885587968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036952858575727183, + "loss": 2.7073, + "theoretical_loss": 3.6920849838537273, + "tokens_seen": 885653504 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369518555667001, + "loss": 2.8973, + "theoretical_loss": 3.6920583983816684, + "tokens_seen": 885719040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003695085255767302, + "loss": 2.797, + "theoretical_loss": 3.6920318154273892, + "tokens_seen": 885784576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694984954864594, + "loss": 2.8963, + "theoretical_loss": 3.6920052349904653, + "tokens_seen": 885850112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036948846539618856, + "loss": 2.6427, + "theoretical_loss": 3.691978657070471, + "tokens_seen": 885915648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036947843530591774, + "loss": 2.7269, + "theoretical_loss": 3.691952081666983, + "tokens_seen": 885981184 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369468405215647, + "loss": 2.745, + "theoretical_loss": 3.691925508779577, + "tokens_seen": 886046720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694583751253761, + "loss": 2.5784, + "theoretical_loss": 3.691898938407828, + "tokens_seen": 886112256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036944834503510534, + "loss": 2.7445, + "theoretical_loss": 3.691872370551312, + "tokens_seen": 886177792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036943831494483446, + "loss": 2.6901, + "theoretical_loss": 3.6918458052096046, + "tokens_seen": 886243328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694282848545637, + "loss": 2.8246, + "theoretical_loss": 3.691819242382283, + "tokens_seen": 886308864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 496242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7018520832061768, + "objective/train/theoretical_loss": 3.6917926820689217, + "objective/train/tokens_used": 906834400, + "theoretical_loss": 3.6917926820689217, + "tokens_seen": 886374400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003694182547642929, + "loss": 2.6236, + "theoretical_loss": 3.6917926820689217, + "tokens_seen": 886374400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036940822467402206, + "loss": 2.6371, + "theoretical_loss": 3.6917661242690984, + "tokens_seen": 886439936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036939819458375124, + "loss": 2.8313, + "theoretical_loss": 3.691739568982388, + "tokens_seen": 886505472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693881644934805, + "loss": 2.8547, + "theoretical_loss": 3.691713016208368, + "tokens_seen": 886571008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693781344032096, + "loss": 2.9052, + "theoretical_loss": 3.691686465946615, + "tokens_seen": 886636544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036936810431293884, + "loss": 2.8262, + "theoretical_loss": 3.691659918196705, + "tokens_seen": 886702080 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369358074222668, + "loss": 2.5036, + "theoretical_loss": 3.691633372958215, + "tokens_seen": 886767616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693480441323972, + "loss": 2.7461, + "theoretical_loss": 3.6916068302307217, + "tokens_seen": 886833152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036933801404212644, + "loss": 2.6719, + "theoretical_loss": 3.6915802900138024, + "tokens_seen": 886898688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036932798395185556, + "loss": 2.8203, + "theoretical_loss": 3.691553752307034, + "tokens_seen": 886964224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693179538615848, + "loss": 2.7727, + "theoretical_loss": 3.6915272171099938, + "tokens_seen": 887029760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003693079237713139, + "loss": 2.5439, + "theoretical_loss": 3.691500684422258, + "tokens_seen": 887095296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036929789368104316, + "loss": 2.6267, + "theoretical_loss": 3.691474154243405, + "tokens_seen": 887160832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036928786359077234, + "loss": 2.8903, + "theoretical_loss": 3.6914476265730123, + "tokens_seen": 887226368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692778335005015, + "loss": 2.8947, + "theoretical_loss": 3.6914211014106573, + "tokens_seen": 887291904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692678034102307, + "loss": 3.0139, + "theoretical_loss": 3.6913945787559177, + "tokens_seen": 887357440 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692577733199599, + "loss": 2.8974, + "theoretical_loss": 3.691368058608371, + "tokens_seen": 887422976 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036924774322968907, + "loss": 2.7185, + "theoretical_loss": 3.6913415409675947, + "tokens_seen": 887488512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003692377131394183, + "loss": 2.7825, + "theoretical_loss": 3.691315025833168, + "tokens_seen": 887554048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036922768304914743, + "loss": 2.737, + "theoretical_loss": 3.6912885132046673, + "tokens_seen": 887619584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036921765295887666, + "loss": 2.7198, + "theoretical_loss": 3.6912620030816723, + "tokens_seen": 887685120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036920762286860585, + "loss": 2.7995, + "theoretical_loss": 3.6912354954637605, + "tokens_seen": 887750656 + }, + { + "epoch": 0.27, + "learning_rate": 0.000369197592778335, + "loss": 2.732, + "theoretical_loss": 3.6912089903505105, + "tokens_seen": 887816192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691875626880642, + "loss": 2.8471, + "theoretical_loss": 3.691182487741501, + "tokens_seen": 887881728 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691775325977934, + "loss": 2.6629, + "theoretical_loss": 3.69115598763631, + "tokens_seen": 887947264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 496686, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.581552267074585, + "objective/train/theoretical_loss": 3.6911294900345166, + "objective/train/tokens_used": 908472800, + "theoretical_loss": 3.6911294900345166, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036916750250752257, + "loss": 2.5457, + "theoretical_loss": 3.6911294900345166, + "tokens_seen": 888012800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691574724172518, + "loss": 2.9441, + "theoretical_loss": 3.691102994935699, + "tokens_seen": 888078336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036914744232698093, + "loss": 2.9934, + "theoretical_loss": 3.6910765023394374, + "tokens_seen": 888143872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036913741223671017, + "loss": 2.813, + "theoretical_loss": 3.6910500122453094, + "tokens_seen": 888209408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691273821464393, + "loss": 2.7331, + "theoretical_loss": 3.691023524652895, + "tokens_seen": 888274944 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036911735205616853, + "loss": 2.5818, + "theoretical_loss": 3.6909970395617724, + "tokens_seen": 888340480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003691073219658977, + "loss": 2.7829, + "theoretical_loss": 3.690970556971522, + "tokens_seen": 888406016 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690972918756269, + "loss": 2.603, + "theoretical_loss": 3.6909440768817223, + "tokens_seen": 888471552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690872617853561, + "loss": 2.7331, + "theoretical_loss": 3.6909175992919536, + "tokens_seen": 888537088 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036907723169508525, + "loss": 2.5585, + "theoretical_loss": 3.690891124201795, + "tokens_seen": 888602624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036906720160481444, + "loss": 2.7386, + "theoretical_loss": 3.690864651610826, + "tokens_seen": 888668160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036905717151454367, + "loss": 2.7889, + "theoretical_loss": 3.690838181518627, + "tokens_seen": 888733696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690471414242728, + "loss": 2.9249, + "theoretical_loss": 3.690811713924778, + "tokens_seen": 888799232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036903711133400203, + "loss": 2.835, + "theoretical_loss": 3.690785248828858, + "tokens_seen": 888864768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690270812437312, + "loss": 2.5885, + "theoretical_loss": 3.690758786230448, + "tokens_seen": 888930304 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690170511534604, + "loss": 2.8022, + "theoretical_loss": 3.690732326129128, + "tokens_seen": 888995840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003690070210631896, + "loss": 2.7309, + "theoretical_loss": 3.690705868524478, + "tokens_seen": 889061376 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036899699097291876, + "loss": 2.8972, + "theoretical_loss": 3.6906794134160785, + "tokens_seen": 889126912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036898696088264794, + "loss": 2.7127, + "theoretical_loss": 3.6906529608035106, + "tokens_seen": 889192448 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689769307923772, + "loss": 2.949, + "theoretical_loss": 3.6906265106863545, + "tokens_seen": 889257984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689669007021063, + "loss": 2.6056, + "theoretical_loss": 3.690600063064191, + "tokens_seen": 889323520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036895687061183554, + "loss": 2.7401, + "theoretical_loss": 3.6905736179366, + "tokens_seen": 889389056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036894684052156466, + "loss": 2.8579, + "theoretical_loss": 3.690547175303164, + "tokens_seen": 889454592 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689368104312939, + "loss": 2.7001, + "theoretical_loss": 3.690520735163463, + "tokens_seen": 889520128 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003689267803410231, + "loss": 2.6667, + "theoretical_loss": 3.6904942975170787, + "tokens_seen": 889585664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 497720, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7590861320495605, + "objective/train/theoretical_loss": 3.6904678623635916, + "objective/train/tokens_used": 910111200, + "theoretical_loss": 3.6904678623635916, + "tokens_seen": 889651200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036891675025075226, + "loss": 2.8205, + "theoretical_loss": 3.6904678623635916, + "tokens_seen": 889651200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036890672016048144, + "loss": 2.5425, + "theoretical_loss": 3.6904414297025836, + "tokens_seen": 889716736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688966900702107, + "loss": 2.9111, + "theoretical_loss": 3.690414999533636, + "tokens_seen": 889782272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688866599799398, + "loss": 2.8207, + "theoretical_loss": 3.69038857185633, + "tokens_seen": 889847808 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036887662988966904, + "loss": 2.5862, + "theoretical_loss": 3.6903621466702474, + "tokens_seen": 889913344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036886659979939817, + "loss": 2.7536, + "theoretical_loss": 3.6903357239749703, + "tokens_seen": 889978880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688565697091274, + "loss": 2.7551, + "theoretical_loss": 3.6903093037700803, + "tokens_seen": 890044416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688465396188566, + "loss": 2.7463, + "theoretical_loss": 3.690282886055159, + "tokens_seen": 890109952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036883650952858576, + "loss": 2.6848, + "theoretical_loss": 3.690256470829789, + "tokens_seen": 890175488 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036882647943831494, + "loss": 2.8369, + "theoretical_loss": 3.690230058093552, + "tokens_seen": 890241024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688164493480441, + "loss": 2.7004, + "theoretical_loss": 3.6902036478460305, + "tokens_seen": 890306560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003688064192577733, + "loss": 2.6664, + "theoretical_loss": 3.6901772400868063, + "tokens_seen": 890372096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036879638916750254, + "loss": 2.6107, + "theoretical_loss": 3.6901508348154626, + "tokens_seen": 890437632 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036878635907723167, + "loss": 2.6259, + "theoretical_loss": 3.6901244320315816, + "tokens_seen": 890503168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687763289869609, + "loss": 2.6725, + "theoretical_loss": 3.6900980317347454, + "tokens_seen": 890568704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036876629889669003, + "loss": 2.6895, + "theoretical_loss": 3.6900716339245383, + "tokens_seen": 890634240 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036875626880641927, + "loss": 2.7007, + "theoretical_loss": 3.6900452386005407, + "tokens_seen": 890699776 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036874623871614845, + "loss": 2.7271, + "theoretical_loss": 3.6900188457623377, + "tokens_seen": 890765312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036873620862587763, + "loss": 2.7241, + "theoretical_loss": 3.689992455409512, + "tokens_seen": 890830848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003687261785356068, + "loss": 2.5983, + "theoretical_loss": 3.6899660675416457, + "tokens_seen": 890896384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036871614844533605, + "loss": 2.9601, + "theoretical_loss": 3.6899396821583226, + "tokens_seen": 890961920 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036870611835506517, + "loss": 2.7535, + "theoretical_loss": 3.6899132992591266, + "tokens_seen": 891027456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686960882647944, + "loss": 2.5921, + "theoretical_loss": 3.68988691884364, + "tokens_seen": 891092992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036868605817452353, + "loss": 2.887, + "theoretical_loss": 3.6898605409114476, + "tokens_seen": 891158528 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036867602808425277, + "loss": 2.6801, + "theoretical_loss": 3.6898341654621314, + "tokens_seen": 891224064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 498750, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7770748138427734, + "objective/train/theoretical_loss": 3.6898077924952775, + "objective/train/tokens_used": 911749600, + "theoretical_loss": 3.6898077924952775, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036866599799398195, + "loss": 2.663, + "theoretical_loss": 3.6898077924952775, + "tokens_seen": 891289600 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036865596790371113, + "loss": 2.883, + "theoretical_loss": 3.6897814220104674, + "tokens_seen": 891355136 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686459378134403, + "loss": 2.7317, + "theoretical_loss": 3.689755054007286, + "tokens_seen": 891420672 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686359077231695, + "loss": 3.0149, + "theoretical_loss": 3.6897286884853173, + "tokens_seen": 891486208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686258776328987, + "loss": 2.7615, + "theoretical_loss": 3.6897023254441463, + "tokens_seen": 891551744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686158475426279, + "loss": 2.6727, + "theoretical_loss": 3.689675964883356, + "tokens_seen": 891617280 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003686058174523571, + "loss": 2.8897, + "theoretical_loss": 3.689649606802531, + "tokens_seen": 891682816 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685957873620863, + "loss": 2.7298, + "theoretical_loss": 3.6896232512012563, + "tokens_seen": 891748352 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036858575727181545, + "loss": 2.6903, + "theoretical_loss": 3.6895968980791163, + "tokens_seen": 891813888 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036857572718154464, + "loss": 2.8455, + "theoretical_loss": 3.689570547435695, + "tokens_seen": 891879424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036856569709127387, + "loss": 2.9195, + "theoretical_loss": 3.6895441992705775, + "tokens_seen": 891944960 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368555667001003, + "loss": 2.9814, + "theoretical_loss": 3.689517853583349, + "tokens_seen": 892010496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036854563691073223, + "loss": 2.5849, + "theoretical_loss": 3.6894915103735944, + "tokens_seen": 892076032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685356068204614, + "loss": 2.847, + "theoretical_loss": 3.689465169640898, + "tokens_seen": 892141568 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685255767301906, + "loss": 2.8258, + "theoretical_loss": 3.6894388313848454, + "tokens_seen": 892207104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003685155466399198, + "loss": 2.6168, + "theoretical_loss": 3.6894124956050223, + "tokens_seen": 892272640 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036850551654964896, + "loss": 2.7964, + "theoretical_loss": 3.6893861623010133, + "tokens_seen": 892338176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036849548645937814, + "loss": 2.6983, + "theoretical_loss": 3.6893598314724043, + "tokens_seen": 892403712 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684854563691074, + "loss": 2.502, + "theoretical_loss": 3.689333503118781, + "tokens_seen": 892469248 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684754262788365, + "loss": 2.8063, + "theoretical_loss": 3.6893071772397286, + "tokens_seen": 892534784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036846539618856574, + "loss": 2.6416, + "theoretical_loss": 3.689280853834833, + "tokens_seen": 892600320 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036845536609829486, + "loss": 2.7694, + "theoretical_loss": 3.6892545329036803, + "tokens_seen": 892665856 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684453360080241, + "loss": 2.3705, + "theoretical_loss": 3.6892282144458557, + "tokens_seen": 892731392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684353059177533, + "loss": 2.7649, + "theoretical_loss": 3.6892018984609463, + "tokens_seen": 892796928 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036842527582748246, + "loss": 2.8509, + "theoretical_loss": 3.6891755849485373, + "tokens_seen": 892862464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 499509, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5587472915649414, + "objective/train/theoretical_loss": 3.6891492739082157, + "objective/train/tokens_used": 913388000, + "theoretical_loss": 3.6891492739082157, + "tokens_seen": 892928000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036841524573721164, + "loss": 2.8054, + "theoretical_loss": 3.6891492739082157, + "tokens_seen": 892928000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003684052156469409, + "loss": 2.5245, + "theoretical_loss": 3.6891229653395667, + "tokens_seen": 892993536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036839518555667, + "loss": 2.39, + "theoretical_loss": 3.6890966592421783, + "tokens_seen": 893059072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036838515546639924, + "loss": 2.5772, + "theoretical_loss": 3.689070355615636, + "tokens_seen": 893124608 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036837512537612837, + "loss": 2.8144, + "theoretical_loss": 3.6890440544595267, + "tokens_seen": 893190144 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683650952858576, + "loss": 2.7312, + "theoretical_loss": 3.6890177557734374, + "tokens_seen": 893255680 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683550651955868, + "loss": 2.8735, + "theoretical_loss": 3.6889914595569544, + "tokens_seen": 893321216 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036834503510531596, + "loss": 2.6178, + "theoretical_loss": 3.688965165809665, + "tokens_seen": 893386752 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036833500501504514, + "loss": 2.7219, + "theoretical_loss": 3.688938874531156, + "tokens_seen": 893452288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683249749247743, + "loss": 2.7928, + "theoretical_loss": 3.688912585721015, + "tokens_seen": 893517824 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003683149448345035, + "loss": 2.6863, + "theoretical_loss": 3.6888862993788285, + "tokens_seen": 893583360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036830491474423274, + "loss": 2.8111, + "theoretical_loss": 3.6888600155041846, + "tokens_seen": 893648896 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036829488465396187, + "loss": 2.7794, + "theoretical_loss": 3.6888337340966704, + "tokens_seen": 893714432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682848545636911, + "loss": 2.7854, + "theoretical_loss": 3.688807455155873, + "tokens_seen": 893779968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036827482447342023, + "loss": 2.8295, + "theoretical_loss": 3.688781178681381, + "tokens_seen": 893845504 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036826479438314947, + "loss": 2.7785, + "theoretical_loss": 3.6887549046727814, + "tokens_seen": 893911040 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036825476429287865, + "loss": 2.7202, + "theoretical_loss": 3.6887286331296627, + "tokens_seen": 893976576 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036824473420260783, + "loss": 2.6292, + "theoretical_loss": 3.6887023640516117, + "tokens_seen": 894042112 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368234704112337, + "loss": 2.5048, + "theoretical_loss": 3.688676097438217, + "tokens_seen": 894107648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036822467402206625, + "loss": 2.7371, + "theoretical_loss": 3.6886498332890674, + "tokens_seen": 894173184 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036821464393179537, + "loss": 2.9452, + "theoretical_loss": 3.68862357160375, + "tokens_seen": 894238720 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003682046138415246, + "loss": 2.7559, + "theoretical_loss": 3.6885973123818543, + "tokens_seen": 894304256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036819458375125373, + "loss": 2.7215, + "theoretical_loss": 3.688571055622967, + "tokens_seen": 894369792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036818455366098297, + "loss": 2.6896, + "theoretical_loss": 3.688544801326679, + "tokens_seen": 894435328 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036817452357071215, + "loss": 2.7963, + "theoretical_loss": 3.688518549492577, + "tokens_seen": 894500864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 500468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6406912803649902, + "objective/train/theoretical_loss": 3.6884923001202505, + "objective/train/tokens_used": 915026400, + "theoretical_loss": 3.6884923001202505, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036816449348044133, + "loss": 2.8942, + "theoretical_loss": 3.6884923001202505, + "tokens_seen": 894566400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681544633901705, + "loss": 2.6235, + "theoretical_loss": 3.6884660532092886, + "tokens_seen": 894631936 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681444332998997, + "loss": 2.8042, + "theoretical_loss": 3.6884398087592793, + "tokens_seen": 894697472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681344032096289, + "loss": 2.82, + "theoretical_loss": 3.688413566769812, + "tokens_seen": 894763008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681243731193581, + "loss": 2.9533, + "theoretical_loss": 3.6883873272404766, + "tokens_seen": 894828544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036811434302908724, + "loss": 2.6947, + "theoretical_loss": 3.6883610901708614, + "tokens_seen": 894894080 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003681043129388165, + "loss": 2.8405, + "theoretical_loss": 3.688334855560556, + "tokens_seen": 894959616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680942828485456, + "loss": 2.8924, + "theoretical_loss": 3.6883086234091493, + "tokens_seen": 895025152 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036808425275827484, + "loss": 2.7066, + "theoretical_loss": 3.688282393716232, + "tokens_seen": 895090688 + }, + { + "epoch": 0.27, + "learning_rate": 0.000368074222668004, + "loss": 2.8829, + "theoretical_loss": 3.688256166481393, + "tokens_seen": 895156224 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680641925777332, + "loss": 2.8899, + "theoretical_loss": 3.688229941704222, + "tokens_seen": 895221760 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680541624874624, + "loss": 2.8019, + "theoretical_loss": 3.6882037193843082, + "tokens_seen": 895287296 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680441323971916, + "loss": 2.4944, + "theoretical_loss": 3.6881774995212426, + "tokens_seen": 895352832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036803410230692074, + "loss": 2.4534, + "theoretical_loss": 3.6881512821146147, + "tokens_seen": 895418368 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036802407221665, + "loss": 2.7225, + "theoretical_loss": 3.688125067164015, + "tokens_seen": 895483904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003680140421263791, + "loss": 2.8461, + "theoretical_loss": 3.6880988546690325, + "tokens_seen": 895549440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036800401203610834, + "loss": 3.0693, + "theoretical_loss": 3.6880726446292593, + "tokens_seen": 895614976 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679939819458375, + "loss": 2.7929, + "theoretical_loss": 3.6880464370442843, + "tokens_seen": 895680512 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679839518555667, + "loss": 2.7852, + "theoretical_loss": 3.6880202319136988, + "tokens_seen": 895746048 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679739217652959, + "loss": 2.7983, + "theoretical_loss": 3.6879940292370934, + "tokens_seen": 895811584 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036796389167502506, + "loss": 2.7209, + "theoretical_loss": 3.687967829014058, + "tokens_seen": 895877120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036795386158475424, + "loss": 2.8093, + "theoretical_loss": 3.6879416312441844, + "tokens_seen": 895942656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679438314944835, + "loss": 2.8308, + "theoretical_loss": 3.6879154359270627, + "tokens_seen": 896008192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679338014042126, + "loss": 2.6901, + "theoretical_loss": 3.6878892430622843, + "tokens_seen": 896073728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036792377131394184, + "loss": 2.8194, + "theoretical_loss": 3.6878630526494405, + "tokens_seen": 896139264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 501526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.16762113571167, + "objective/train/theoretical_loss": 3.687836864688122, + "objective/train/tokens_used": 916664800, + "theoretical_loss": 3.687836864688122, + "tokens_seen": 896204800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036791374122367097, + "loss": 3.0439, + "theoretical_loss": 3.687836864688122, + "tokens_seen": 896204800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003679037111334002, + "loss": 2.8419, + "theoretical_loss": 3.68781067917792, + "tokens_seen": 896270336 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678936810431294, + "loss": 3.0514, + "theoretical_loss": 3.687784496118426, + "tokens_seen": 896335872 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036788365095285857, + "loss": 2.8178, + "theoretical_loss": 3.687758315509232, + "tokens_seen": 896401408 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036787362086258775, + "loss": 2.7582, + "theoretical_loss": 3.687732137349929, + "tokens_seen": 896466944 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367863590772317, + "loss": 3.0367, + "theoretical_loss": 3.687705961640109, + "tokens_seen": 896532480 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036785356068204616, + "loss": 2.7561, + "theoretical_loss": 3.687679788379364, + "tokens_seen": 896598016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036784353059177535, + "loss": 2.6126, + "theoretical_loss": 3.6876536175672854, + "tokens_seen": 896663552 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678335005015045, + "loss": 2.9616, + "theoretical_loss": 3.687627449203465, + "tokens_seen": 896729088 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003678234704112337, + "loss": 2.7894, + "theoretical_loss": 3.6876012832874956, + "tokens_seen": 896794624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036781344032096294, + "loss": 2.7923, + "theoretical_loss": 3.6875751198189683, + "tokens_seen": 896860160 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036780341023069207, + "loss": 2.7743, + "theoretical_loss": 3.687548958797476, + "tokens_seen": 896925696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677933801404213, + "loss": 2.873, + "theoretical_loss": 3.6875228002226113, + "tokens_seen": 896991232 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036778335005015043, + "loss": 2.7466, + "theoretical_loss": 3.6874966440939665, + "tokens_seen": 897056768 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036777331995987967, + "loss": 2.9035, + "theoretical_loss": 3.6874704904111337, + "tokens_seen": 897122304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036776328986960885, + "loss": 2.9417, + "theoretical_loss": 3.687444339173706, + "tokens_seen": 897187840 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036775325977933803, + "loss": 2.7169, + "theoretical_loss": 3.687418190381276, + "tokens_seen": 897253376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677432296890672, + "loss": 2.6652, + "theoretical_loss": 3.6873920440334373, + "tokens_seen": 897318912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036773319959879645, + "loss": 2.402, + "theoretical_loss": 3.687365900129781, + "tokens_seen": 897384448 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036772316950852557, + "loss": 2.8106, + "theoretical_loss": 3.6873397586699026, + "tokens_seen": 897449984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003677131394182548, + "loss": 2.7959, + "theoretical_loss": 3.687313619653393, + "tokens_seen": 897515520 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036770310932798393, + "loss": 2.6755, + "theoretical_loss": 3.6872874830798463, + "tokens_seen": 897581056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036769307923771317, + "loss": 2.8384, + "theoretical_loss": 3.6872613489488564, + "tokens_seen": 897646592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036768304914744235, + "loss": 2.7293, + "theoretical_loss": 3.6872352172600156, + "tokens_seen": 897712128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036767301905717153, + "loss": 2.5484, + "theoretical_loss": 3.6872090880129185, + "tokens_seen": 897777664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 502221, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.841933012008667, + "objective/train/theoretical_loss": 3.6871829612071583, + "objective/train/tokens_used": 918303200, + "theoretical_loss": 3.6871829612071583, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676629889669007, + "loss": 2.6498, + "theoretical_loss": 3.6871829612071583, + "tokens_seen": 897843200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676529588766299, + "loss": 2.8582, + "theoretical_loss": 3.6871568368423286, + "tokens_seen": 897908736 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676429287863591, + "loss": 2.7102, + "theoretical_loss": 3.6871307149180232, + "tokens_seen": 897974272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676328986960883, + "loss": 2.7657, + "theoretical_loss": 3.687104595433836, + "tokens_seen": 898039808 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036762286860581744, + "loss": 2.7004, + "theoretical_loss": 3.6870784783893615, + "tokens_seen": 898105344 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676128385155467, + "loss": 2.7002, + "theoretical_loss": 3.6870523637841934, + "tokens_seen": 898170880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003676028084252758, + "loss": 2.6307, + "theoretical_loss": 3.687026251617926, + "tokens_seen": 898236416 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036759277833500504, + "loss": 2.877, + "theoretical_loss": 3.6870001418901532, + "tokens_seen": 898301952 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675827482447342, + "loss": 2.9289, + "theoretical_loss": 3.6869740346004702, + "tokens_seen": 898367488 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675727181544634, + "loss": 2.7154, + "theoretical_loss": 3.686947929748471, + "tokens_seen": 898433024 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675626880641926, + "loss": 3.0352, + "theoretical_loss": 3.6869218273337503, + "tokens_seen": 898498560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675526579739218, + "loss": 2.8265, + "theoretical_loss": 3.6868957273559024, + "tokens_seen": 898564096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036754262788365094, + "loss": 2.6395, + "theoretical_loss": 3.686869629814523, + "tokens_seen": 898629632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675325977933802, + "loss": 2.8067, + "theoretical_loss": 3.6868435347092063, + "tokens_seen": 898695168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675225677031093, + "loss": 2.7041, + "theoretical_loss": 3.686817442039547, + "tokens_seen": 898760704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036751253761283854, + "loss": 2.8153, + "theoretical_loss": 3.6867913518051414, + "tokens_seen": 898826240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003675025075225677, + "loss": 2.6424, + "theoretical_loss": 3.6867652640055835, + "tokens_seen": 898891776 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674924774322969, + "loss": 2.8134, + "theoretical_loss": 3.686739178640469, + "tokens_seen": 898957312 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674824473420261, + "loss": 2.6906, + "theoretical_loss": 3.6867130957093934, + "tokens_seen": 899022848 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036747241725175526, + "loss": 2.707, + "theoretical_loss": 3.6866870152119517, + "tokens_seen": 899088384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036746238716148444, + "loss": 2.9336, + "theoretical_loss": 3.6866609371477397, + "tokens_seen": 899153920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674523570712137, + "loss": 2.5698, + "theoretical_loss": 3.6866348615163536, + "tokens_seen": 899219456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674423269809428, + "loss": 2.7786, + "theoretical_loss": 3.686608788317389, + "tokens_seen": 899284992 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036743229689067204, + "loss": 2.795, + "theoretical_loss": 3.686582717550441, + "tokens_seen": 899350528 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036742226680040117, + "loss": 2.8089, + "theoretical_loss": 3.6865566492151056, + "tokens_seen": 899416064 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 503170, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.086702585220337, + "objective/train/theoretical_loss": 3.6865305833109794, + "objective/train/tokens_used": 919941600, + "theoretical_loss": 3.6865305833109794, + "tokens_seen": 899481600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674122367101304, + "loss": 2.8526, + "theoretical_loss": 3.6865305833109794, + "tokens_seen": 899481600 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003674022066198596, + "loss": 2.7103, + "theoretical_loss": 3.6865045198376594, + "tokens_seen": 899547136 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036739217652958877, + "loss": 2.5794, + "theoretical_loss": 3.6864784587947397, + "tokens_seen": 899612672 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036738214643931795, + "loss": 2.8274, + "theoretical_loss": 3.6864524001818184, + "tokens_seen": 899678208 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673721163490472, + "loss": 2.7288, + "theoretical_loss": 3.686426343998491, + "tokens_seen": 899743744 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673620862587763, + "loss": 2.7591, + "theoretical_loss": 3.6864002902443547, + "tokens_seen": 899809280 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036735205616850555, + "loss": 2.5409, + "theoretical_loss": 3.6863742389190053, + "tokens_seen": 899874816 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036734202607823467, + "loss": 2.87, + "theoretical_loss": 3.6863481900220405, + "tokens_seen": 899940352 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673319959879639, + "loss": 2.6941, + "theoretical_loss": 3.6863221435530567, + "tokens_seen": 900005888 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003673219658976931, + "loss": 3.0026, + "theoretical_loss": 3.6862960995116505, + "tokens_seen": 900071424 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036731193580742227, + "loss": 2.6116, + "theoretical_loss": 3.6862700578974192, + "tokens_seen": 900136960 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036730190571715145, + "loss": 2.8755, + "theoretical_loss": 3.68624401870996, + "tokens_seen": 900202496 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036729187562688063, + "loss": 2.4658, + "theoretical_loss": 3.68621798194887, + "tokens_seen": 900268032 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672818455366098, + "loss": 2.8991, + "theoretical_loss": 3.686191947613746, + "tokens_seen": 900333568 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036727181544633905, + "loss": 2.972, + "theoretical_loss": 3.686165915704186, + "tokens_seen": 900399104 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672617853560682, + "loss": 2.9544, + "theoretical_loss": 3.686139886219788, + "tokens_seen": 900464640 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672517552657974, + "loss": 2.7917, + "theoretical_loss": 3.6861138591601486, + "tokens_seen": 900530176 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036724172517552654, + "loss": 2.7219, + "theoretical_loss": 3.6860878345248658, + "tokens_seen": 900595712 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036723169508525577, + "loss": 2.7, + "theoretical_loss": 3.686061812313538, + "tokens_seen": 900661248 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036722166499498495, + "loss": 2.9898, + "theoretical_loss": 3.686035792525762, + "tokens_seen": 900726784 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036721163490471414, + "loss": 2.7812, + "theoretical_loss": 3.6860097751611356, + "tokens_seen": 900792320 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003672016048144433, + "loss": 2.7424, + "theoretical_loss": 3.6859837602192584, + "tokens_seen": 900857856 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036719157472417255, + "loss": 2.9703, + "theoretical_loss": 3.6859577476997276, + "tokens_seen": 900923392 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671815446339017, + "loss": 2.8666, + "theoretical_loss": 3.6859317376021417, + "tokens_seen": 900988928 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671715145436309, + "loss": 2.7763, + "theoretical_loss": 3.6859057299260987, + "tokens_seen": 901054464 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 504328, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.588376045227051, + "objective/train/theoretical_loss": 3.6858797246711976, + "objective/train/tokens_used": 921580000, + "theoretical_loss": 3.6858797246711976, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036716148445336004, + "loss": 2.7931, + "theoretical_loss": 3.6858797246711976, + "tokens_seen": 901120000 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671514543630893, + "loss": 2.7644, + "theoretical_loss": 3.685853721837036, + "tokens_seen": 901185536 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036714142427281846, + "loss": 2.8663, + "theoretical_loss": 3.6858277214232142, + "tokens_seen": 901251072 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036713139418254764, + "loss": 2.8726, + "theoretical_loss": 3.6858017234293294, + "tokens_seen": 901316608 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003671213640922768, + "loss": 2.8011, + "theoretical_loss": 3.6857757278549803, + "tokens_seen": 901382144 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367111334002006, + "loss": 2.9276, + "theoretical_loss": 3.6857497346997676, + "tokens_seen": 901447680 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036710130391173524, + "loss": 2.8506, + "theoretical_loss": 3.6857237439632886, + "tokens_seen": 901513216 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670912738214644, + "loss": 2.8113, + "theoretical_loss": 3.685697755645143, + "tokens_seen": 901578752 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670812437311936, + "loss": 2.7378, + "theoretical_loss": 3.6856717697449306, + "tokens_seen": 901644288 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670712136409228, + "loss": 2.7982, + "theoretical_loss": 3.68564578626225, + "tokens_seen": 901709824 + }, + { + "epoch": 0.27, + "learning_rate": 0.000367061183550652, + "loss": 2.3642, + "theoretical_loss": 3.6856198051967004, + "tokens_seen": 901775360 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036705115346038114, + "loss": 2.837, + "theoretical_loss": 3.6855938265478825, + "tokens_seen": 901840896 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670411233701104, + "loss": 2.7009, + "theoretical_loss": 3.6855678503153944, + "tokens_seen": 901906432 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670310932798395, + "loss": 2.8224, + "theoretical_loss": 3.685541876498837, + "tokens_seen": 901971968 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036702106318956874, + "loss": 2.8333, + "theoretical_loss": 3.685515905097809, + "tokens_seen": 902037504 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670110330992979, + "loss": 2.8069, + "theoretical_loss": 3.685489936111911, + "tokens_seen": 902103040 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003670010030090271, + "loss": 2.8903, + "theoretical_loss": 3.685463969540743, + "tokens_seen": 902168576 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669909729187563, + "loss": 2.6957, + "theoretical_loss": 3.685438005383905, + "tokens_seen": 902234112 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036698094282848546, + "loss": 2.7104, + "theoretical_loss": 3.685412043640997, + "tokens_seen": 902299648 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036697091273821464, + "loss": 2.7687, + "theoretical_loss": 3.6853860843116193, + "tokens_seen": 902365184 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669608826479439, + "loss": 2.8506, + "theoretical_loss": 3.6853601273953727, + "tokens_seen": 902430720 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366950852557673, + "loss": 2.6235, + "theoretical_loss": 3.6853341728918574, + "tokens_seen": 902496256 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036694082246740224, + "loss": 2.8101, + "theoretical_loss": 3.6853082208006733, + "tokens_seen": 902561792 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036693079237713137, + "loss": 2.9473, + "theoretical_loss": 3.685282271121422, + "tokens_seen": 902627328 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669207622868606, + "loss": 2.7178, + "theoretical_loss": 3.6852563238537037, + "tokens_seen": 902692864 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 504856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7679882049560547, + "objective/train/theoretical_loss": 3.6852303789971192, + "objective/train/tokens_used": 923218400, + "theoretical_loss": 3.6852303789971192, + "tokens_seen": 902758400 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003669107321965898, + "loss": 2.8659, + "theoretical_loss": 3.6852303789971192, + "tokens_seen": 902758400 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036690070210631897, + "loss": 2.9552, + "theoretical_loss": 3.68520443655127, + "tokens_seen": 902823936 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036689067201604815, + "loss": 2.63, + "theoretical_loss": 3.6851784965157566, + "tokens_seen": 902889472 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668806419257774, + "loss": 2.665, + "theoretical_loss": 3.68515255889018, + "tokens_seen": 902955008 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668706118355065, + "loss": 2.7384, + "theoretical_loss": 3.685126623674142, + "tokens_seen": 903020544 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036686058174523575, + "loss": 2.8029, + "theoretical_loss": 3.6851006908672437, + "tokens_seen": 903086080 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036685055165496487, + "loss": 2.7919, + "theoretical_loss": 3.685074760469086, + "tokens_seen": 903151616 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668405215646941, + "loss": 2.74, + "theoretical_loss": 3.6850488324792714, + "tokens_seen": 903217152 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003668304914744233, + "loss": 2.8418, + "theoretical_loss": 3.6850229068974008, + "tokens_seen": 903282688 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036682046138415247, + "loss": 2.7863, + "theoretical_loss": 3.6849969837230754, + "tokens_seen": 903348224 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036681043129388165, + "loss": 2.8803, + "theoretical_loss": 3.6849710629558983, + "tokens_seen": 903413760 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036680040120361083, + "loss": 2.9019, + "theoretical_loss": 3.6849451445954706, + "tokens_seen": 903479296 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036679037111334, + "loss": 2.836, + "theoretical_loss": 3.684919228641394, + "tokens_seen": 903544832 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036678034102306925, + "loss": 2.7521, + "theoretical_loss": 3.6848933150932712, + "tokens_seen": 903610368 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667703109327984, + "loss": 2.5911, + "theoretical_loss": 3.684867403950704, + "tokens_seen": 903675904 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667602808425276, + "loss": 2.6519, + "theoretical_loss": 3.684841495213295, + "tokens_seen": 903741440 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036675025075225674, + "loss": 2.8198, + "theoretical_loss": 3.684815588880646, + "tokens_seen": 903806976 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366740220661986, + "loss": 2.5777, + "theoretical_loss": 3.68478968495236, + "tokens_seen": 903872512 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036673019057171515, + "loss": 2.875, + "theoretical_loss": 3.684763783428039, + "tokens_seen": 903938048 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036672016048144434, + "loss": 2.7931, + "theoretical_loss": 3.684737884307286, + "tokens_seen": 904003584 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003667101303911735, + "loss": 2.6516, + "theoretical_loss": 3.6847119875897043, + "tokens_seen": 904069120 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036670010030090275, + "loss": 2.8678, + "theoretical_loss": 3.684686093274896, + "tokens_seen": 904134656 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666900702106319, + "loss": 2.6876, + "theoretical_loss": 3.6846602013624636, + "tokens_seen": 904200192 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666800401203611, + "loss": 2.5903, + "theoretical_loss": 3.684634311852011, + "tokens_seen": 904265728 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036667001003009024, + "loss": 2.8271, + "theoretical_loss": 3.684608424743141, + "tokens_seen": 904331264 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 505908, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.720822811126709, + "objective/train/theoretical_loss": 3.684582540035456, + "objective/train/tokens_used": 924856800, + "theoretical_loss": 3.684582540035456, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666599799398195, + "loss": 2.7747, + "theoretical_loss": 3.684582540035456, + "tokens_seen": 904396800 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036664994984954866, + "loss": 2.5759, + "theoretical_loss": 3.684556657728561, + "tokens_seen": 904462336 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036663991975927784, + "loss": 2.8172, + "theoretical_loss": 3.684530777822059, + "tokens_seen": 904527872 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366629889669007, + "loss": 2.6142, + "theoretical_loss": 3.684504900315552, + "tokens_seen": 904593408 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666198595787362, + "loss": 2.8247, + "theoretical_loss": 3.6844790252086446, + "tokens_seen": 904658944 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003666098294884654, + "loss": 2.7764, + "theoretical_loss": 3.6844531525009403, + "tokens_seen": 904724480 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665997993981946, + "loss": 2.8979, + "theoretical_loss": 3.6844272821920434, + "tokens_seen": 904790016 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036658976930792374, + "loss": 2.7468, + "theoretical_loss": 3.6844014142815578, + "tokens_seen": 904855552 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366579739217653, + "loss": 2.9656, + "theoretical_loss": 3.6843755487690864, + "tokens_seen": 904921088 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036656970912738216, + "loss": 2.6941, + "theoretical_loss": 3.684349685654234, + "tokens_seen": 904986624 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036655967903711134, + "loss": 2.6184, + "theoretical_loss": 3.6843238249366044, + "tokens_seen": 905052160 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665496489468405, + "loss": 2.659, + "theoretical_loss": 3.6842979666158024, + "tokens_seen": 905117696 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665396188565697, + "loss": 2.9313, + "theoretical_loss": 3.6842721106914316, + "tokens_seen": 905183232 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665295887662989, + "loss": 2.6039, + "theoretical_loss": 3.6842462571630974, + "tokens_seen": 905248768 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003665195586760281, + "loss": 3.0197, + "theoretical_loss": 3.6842204060304033, + "tokens_seen": 905314304 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036650952858575725, + "loss": 2.7065, + "theoretical_loss": 3.6841945572929546, + "tokens_seen": 905379840 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664994984954865, + "loss": 2.6915, + "theoretical_loss": 3.684168710950356, + "tokens_seen": 905445376 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664894684052156, + "loss": 2.6842, + "theoretical_loss": 3.684142867002212, + "tokens_seen": 905510912 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036647943831494484, + "loss": 2.6965, + "theoretical_loss": 3.684117025448127, + "tokens_seen": 905576448 + }, + { + "epoch": 0.27, + "learning_rate": 0.000366469408224674, + "loss": 2.9644, + "theoretical_loss": 3.684091186287707, + "tokens_seen": 905641984 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664593781344032, + "loss": 2.8037, + "theoretical_loss": 3.684065349520557, + "tokens_seen": 905707520 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664493480441324, + "loss": 2.5095, + "theoretical_loss": 3.6840395151462815, + "tokens_seen": 905773056 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036643931795386157, + "loss": 2.6812, + "theoretical_loss": 3.684013683164486, + "tokens_seen": 905838592 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036642928786359075, + "loss": 2.852, + "theoretical_loss": 3.6839878535747763, + "tokens_seen": 905904128 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036641925777332, + "loss": 2.6574, + "theoretical_loss": 3.683962026376758, + "tokens_seen": 905969664 + }, + { + "epoch": 0.27, + "objective/train/docs_used": 506404, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0108041763305664, + "objective/train/theoretical_loss": 3.6839362015700354, + "objective/train/tokens_used": 926495200, + "theoretical_loss": 3.6839362015700354, + "tokens_seen": 906035200 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003664092276830491, + "loss": 2.5565, + "theoretical_loss": 3.6839362015700354, + "tokens_seen": 906035200 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036639919759277835, + "loss": 2.7534, + "theoretical_loss": 3.683910379154215, + "tokens_seen": 906100736 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036638916750250753, + "loss": 2.4499, + "theoretical_loss": 3.683884559128903, + "tokens_seen": 906166272 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663791374122367, + "loss": 2.7014, + "theoretical_loss": 3.6838587414937045, + "tokens_seen": 906231808 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663691073219659, + "loss": 2.6931, + "theoretical_loss": 3.6838329262482254, + "tokens_seen": 906297344 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036635907723169507, + "loss": 2.9389, + "theoretical_loss": 3.6838071133920725, + "tokens_seen": 906362880 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663490471414243, + "loss": 2.7749, + "theoretical_loss": 3.683781302924851, + "tokens_seen": 906428416 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003663390170511535, + "loss": 2.7101, + "theoretical_loss": 3.6837554948461677, + "tokens_seen": 906493952 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036632898696088267, + "loss": 2.6792, + "theoretical_loss": 3.683729689155629, + "tokens_seen": 906559488 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036631895687061185, + "loss": 2.7912, + "theoretical_loss": 3.6837038858528404, + "tokens_seen": 906625024 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036630892678034103, + "loss": 2.728, + "theoretical_loss": 3.6836780849374096, + "tokens_seen": 906690560 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662988966900702, + "loss": 2.6709, + "theoretical_loss": 3.6836522864089423, + "tokens_seen": 906756096 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036628886659979945, + "loss": 2.8336, + "theoretical_loss": 3.683626490267046, + "tokens_seen": 906821632 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662788365095286, + "loss": 2.7861, + "theoretical_loss": 3.683600696511326, + "tokens_seen": 906887168 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662688064192578, + "loss": 2.8822, + "theoretical_loss": 3.6835749051413904, + "tokens_seen": 906952704 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036625877632898694, + "loss": 2.8046, + "theoretical_loss": 3.683549116156846, + "tokens_seen": 907018240 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662487462387162, + "loss": 2.6543, + "theoretical_loss": 3.6835233295573, + "tokens_seen": 907083776 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036623871614844535, + "loss": 2.9645, + "theoretical_loss": 3.6834975453423593, + "tokens_seen": 907149312 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036622868605817454, + "loss": 2.7094, + "theoretical_loss": 3.6834717635116307, + "tokens_seen": 907214848 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003662186559679037, + "loss": 2.7114, + "theoretical_loss": 3.683445984064722, + "tokens_seen": 907280384 + }, + { + "epoch": 0.27, + "learning_rate": 0.00036620862587763295, + "loss": 2.6491, + "theoretical_loss": 3.6834202070012405, + "tokens_seen": 907345920 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003661985957873621, + "loss": 2.8925, + "theoretical_loss": 3.6833944323207937, + "tokens_seen": 907411456 + }, + { + "epoch": 0.27, + "learning_rate": 0.0003661885656970913, + "loss": 2.9666, + "theoretical_loss": 3.68336866002299, + "tokens_seen": 907476992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036617853560682044, + "loss": 2.6134, + "theoretical_loss": 3.6833428901074354, + "tokens_seen": 907542528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661685055165497, + "loss": 2.7143, + "theoretical_loss": 3.683317122573739, + "tokens_seen": 907608064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 507582, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5680878162384033, + "objective/train/theoretical_loss": 3.683291357421508, + "objective/train/tokens_used": 928133600, + "theoretical_loss": 3.683291357421508, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036615847542627886, + "loss": 2.7327, + "theoretical_loss": 3.683291357421508, + "tokens_seen": 907673600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036614844533600804, + "loss": 2.8497, + "theoretical_loss": 3.683265594650351, + "tokens_seen": 907739136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661384152457372, + "loss": 2.6495, + "theoretical_loss": 3.683239834259876, + "tokens_seen": 907804672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661283851554664, + "loss": 2.9766, + "theoretical_loss": 3.6832140762496905, + "tokens_seen": 907870208 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661183550651956, + "loss": 2.7795, + "theoretical_loss": 3.683188320619404, + "tokens_seen": 907935744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003661083249749248, + "loss": 2.6712, + "theoretical_loss": 3.6831625673686235, + "tokens_seen": 908001280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036609829488465394, + "loss": 2.7726, + "theoretical_loss": 3.6831368164969582, + "tokens_seen": 908066816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660882647943832, + "loss": 2.7, + "theoretical_loss": 3.683111068004016, + "tokens_seen": 908132352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036607823470411236, + "loss": 2.6533, + "theoretical_loss": 3.6830853218894064, + "tokens_seen": 908197888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036606820461384154, + "loss": 2.6621, + "theoretical_loss": 3.6830595781527373, + "tokens_seen": 908263424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660581745235707, + "loss": 2.8009, + "theoretical_loss": 3.6830338367936184, + "tokens_seen": 908328960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660481444332999, + "loss": 2.6356, + "theoretical_loss": 3.683008097811658, + "tokens_seen": 908394496 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660381143430291, + "loss": 2.8399, + "theoretical_loss": 3.6829823612064647, + "tokens_seen": 908460032 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660280842527583, + "loss": 2.8992, + "theoretical_loss": 3.6829566269776484, + "tokens_seen": 908525568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036601805416248745, + "loss": 2.7555, + "theoretical_loss": 3.6829308951248185, + "tokens_seen": 908591104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003660080240722167, + "loss": 2.9406, + "theoretical_loss": 3.682905165647583, + "tokens_seen": 908656640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659979939819458, + "loss": 2.7128, + "theoretical_loss": 3.6828794385455526, + "tokens_seen": 908722176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036598796389167504, + "loss": 2.6451, + "theoretical_loss": 3.6828537138183357, + "tokens_seen": 908787712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659779338014042, + "loss": 2.8677, + "theoretical_loss": 3.682827991465543, + "tokens_seen": 908853248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659679037111334, + "loss": 2.7044, + "theoretical_loss": 3.682802271486783, + "tokens_seen": 908918784 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659578736208626, + "loss": 2.7429, + "theoretical_loss": 3.6827765538816655, + "tokens_seen": 908984320 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036594784353059177, + "loss": 2.8765, + "theoretical_loss": 3.6827508386498016, + "tokens_seen": 909049856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036593781344032095, + "loss": 2.6742, + "theoretical_loss": 3.6827251257907996, + "tokens_seen": 909115392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659277833500502, + "loss": 2.651, + "theoretical_loss": 3.682699415304271, + "tokens_seen": 909180928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003659177532597793, + "loss": 2.6706, + "theoretical_loss": 3.682673707189825, + "tokens_seen": 909246464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 508185, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.757549524307251, + "objective/train/theoretical_loss": 3.6826480014470717, + "objective/train/tokens_used": 929772000, + "theoretical_loss": 3.6826480014470717, + "tokens_seen": 909312000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036590772316950855, + "loss": 2.6325, + "theoretical_loss": 3.6826480014470717, + "tokens_seen": 909312000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036589769307923773, + "loss": 2.5434, + "theoretical_loss": 3.6826222980756222, + "tokens_seen": 909377536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658876629889669, + "loss": 2.9387, + "theoretical_loss": 3.6825965970750856, + "tokens_seen": 909443072 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658776328986961, + "loss": 2.8113, + "theoretical_loss": 3.682570898445074, + "tokens_seen": 909508608 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036586760280842527, + "loss": 2.8491, + "theoretical_loss": 3.6825452021851963, + "tokens_seen": 909574144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036585757271815445, + "loss": 2.6226, + "theoretical_loss": 3.682519508295065, + "tokens_seen": 909639680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658475426278837, + "loss": 2.4749, + "theoretical_loss": 3.6824938167742887, + "tokens_seen": 909705216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658375125376128, + "loss": 2.7062, + "theoretical_loss": 3.6824681276224798, + "tokens_seen": 909770752 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036582748244734205, + "loss": 2.5519, + "theoretical_loss": 3.682442440839249, + "tokens_seen": 909836288 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658174523570712, + "loss": 2.7983, + "theoretical_loss": 3.6824167564242067, + "tokens_seen": 909901824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003658074222668004, + "loss": 2.7716, + "theoretical_loss": 3.682391074376965, + "tokens_seen": 909967360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657973921765296, + "loss": 2.7686, + "theoretical_loss": 3.6823653946971344, + "tokens_seen": 910032896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657873620862588, + "loss": 2.6568, + "theoretical_loss": 3.682339717384326, + "tokens_seen": 910098432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036577733199598796, + "loss": 2.7357, + "theoretical_loss": 3.6823140424381524, + "tokens_seen": 910163968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036576730190571714, + "loss": 2.7161, + "theoretical_loss": 3.682288369858223, + "tokens_seen": 910229504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657572718154463, + "loss": 2.7993, + "theoretical_loss": 3.6822626996441516, + "tokens_seen": 910295040 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036574724172517555, + "loss": 2.9071, + "theoretical_loss": 3.6822370317955486, + "tokens_seen": 910360576 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657372116349047, + "loss": 2.5976, + "theoretical_loss": 3.682211366312026, + "tokens_seen": 910426112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657271815446339, + "loss": 2.7127, + "theoretical_loss": 3.6821857031931957, + "tokens_seen": 910491648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657171514543631, + "loss": 2.7244, + "theoretical_loss": 3.68216004243867, + "tokens_seen": 910557184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003657071213640923, + "loss": 2.768, + "theoretical_loss": 3.6821343840480605, + "tokens_seen": 910622720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036569709127382146, + "loss": 2.6111, + "theoretical_loss": 3.682108728020979, + "tokens_seen": 910688256 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036568706118355064, + "loss": 2.8181, + "theoretical_loss": 3.6820830743570383, + "tokens_seen": 910753792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656770310932798, + "loss": 2.7025, + "theoretical_loss": 3.6820574230558503, + "tokens_seen": 910819328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036566700100300906, + "loss": 2.7488, + "theoretical_loss": 3.682031774117028, + "tokens_seen": 910884864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 509502, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5731163024902344, + "objective/train/theoretical_loss": 3.682006127540184, + "objective/train/tokens_used": 931410400, + "theoretical_loss": 3.682006127540184, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656569709127382, + "loss": 2.7661, + "theoretical_loss": 3.682006127540184, + "tokens_seen": 910950400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656469408224674, + "loss": 2.9085, + "theoretical_loss": 3.6819804833249297, + "tokens_seen": 911015936 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036563691073219655, + "loss": 2.7918, + "theoretical_loss": 3.681954841470879, + "tokens_seen": 911081472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003656268806419258, + "loss": 2.6731, + "theoretical_loss": 3.6819292019776437, + "tokens_seen": 911147008 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036561685055165496, + "loss": 2.7863, + "theoretical_loss": 3.6819035648448373, + "tokens_seen": 911212544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036560682046138414, + "loss": 2.8253, + "theoretical_loss": 3.6818779300720728, + "tokens_seen": 911278080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655967903711134, + "loss": 2.7498, + "theoretical_loss": 3.6818522976589625, + "tokens_seen": 911343616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036558676028084256, + "loss": 2.6159, + "theoretical_loss": 3.6818266676051206, + "tokens_seen": 911409152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036557673019057174, + "loss": 2.7005, + "theoretical_loss": 3.6818010399101597, + "tokens_seen": 911474688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655667001003009, + "loss": 2.6145, + "theoretical_loss": 3.6817754145736927, + "tokens_seen": 911540224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655566700100301, + "loss": 2.7677, + "theoretical_loss": 3.681749791595334, + "tokens_seen": 911605760 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655466399197593, + "loss": 2.9102, + "theoretical_loss": 3.6817241709746966, + "tokens_seen": 911671296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655366098294885, + "loss": 2.8622, + "theoretical_loss": 3.681698552711394, + "tokens_seen": 911736832 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036552657973921765, + "loss": 2.7773, + "theoretical_loss": 3.6816729368050396, + "tokens_seen": 911802368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003655165496489469, + "loss": 2.6455, + "theoretical_loss": 3.6816473232552482, + "tokens_seen": 911867904 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365506519558676, + "loss": 2.699, + "theoretical_loss": 3.6816217120616326, + "tokens_seen": 911933440 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036549648946840525, + "loss": 2.8395, + "theoretical_loss": 3.681596103223807, + "tokens_seen": 911998976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654864593781344, + "loss": 2.7838, + "theoretical_loss": 3.6815704967413865, + "tokens_seen": 912064512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654764292878636, + "loss": 2.7302, + "theoretical_loss": 3.6815448926139833, + "tokens_seen": 912130048 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654663991975928, + "loss": 2.8579, + "theoretical_loss": 3.6815192908412135, + "tokens_seen": 912195584 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036545636910732197, + "loss": 2.7153, + "theoretical_loss": 3.6814936914226903, + "tokens_seen": 912261120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036544633901705115, + "loss": 2.8546, + "theoretical_loss": 3.6814680943580287, + "tokens_seen": 912326656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654363089267804, + "loss": 2.5904, + "theoretical_loss": 3.681442499646842, + "tokens_seen": 912392192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003654262788365095, + "loss": 2.6852, + "theoretical_loss": 3.6814169072887464, + "tokens_seen": 912457728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036541624874623875, + "loss": 2.6125, + "theoretical_loss": 3.6813913172833557, + "tokens_seen": 912523264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 510138, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.649203062057495, + "objective/train/theoretical_loss": 3.6813657296302846, + "objective/train/tokens_used": 933048800, + "theoretical_loss": 3.6813657296302846, + "tokens_seen": 912588800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036540621865596793, + "loss": 2.6317, + "theoretical_loss": 3.6813657296302846, + "tokens_seen": 912588800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653961885656971, + "loss": 2.6273, + "theoretical_loss": 3.6813401443291482, + "tokens_seen": 912654336 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653861584754263, + "loss": 2.8201, + "theoretical_loss": 3.681314561379562, + "tokens_seen": 912719872 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036537612838515547, + "loss": 2.8762, + "theoretical_loss": 3.68128898078114, + "tokens_seen": 912785408 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036536609829488465, + "loss": 2.7939, + "theoretical_loss": 3.6812634025334985, + "tokens_seen": 912850944 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653560682046139, + "loss": 2.7877, + "theoretical_loss": 3.6812378266362513, + "tokens_seen": 912916480 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365346038114343, + "loss": 2.7212, + "theoretical_loss": 3.6812122530890146, + "tokens_seen": 912982016 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036533600802407225, + "loss": 2.683, + "theoretical_loss": 3.681186681891404, + "tokens_seen": 913047552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653259779338014, + "loss": 2.7768, + "theoretical_loss": 3.6811611130430344, + "tokens_seen": 913113088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653159478435306, + "loss": 2.6558, + "theoretical_loss": 3.681135546543522, + "tokens_seen": 913178624 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003653059177532598, + "loss": 2.6317, + "theoretical_loss": 3.6811099823924813, + "tokens_seen": 913244160 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365295887662989, + "loss": 2.7281, + "theoretical_loss": 3.6810844205895297, + "tokens_seen": 913309696 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036528585757271816, + "loss": 2.6293, + "theoretical_loss": 3.6810588611342823, + "tokens_seen": 913375232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036527582748244734, + "loss": 2.7732, + "theoretical_loss": 3.6810333040263545, + "tokens_seen": 913440768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652657973921765, + "loss": 2.7447, + "theoretical_loss": 3.6810077492653632, + "tokens_seen": 913506304 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036525576730190575, + "loss": 2.8119, + "theoretical_loss": 3.680982196850924, + "tokens_seen": 913571840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652457372116349, + "loss": 2.6916, + "theoretical_loss": 3.6809566467826533, + "tokens_seen": 913637376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652357071213641, + "loss": 2.8739, + "theoretical_loss": 3.680931099060168, + "tokens_seen": 913702912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652256770310933, + "loss": 2.6952, + "theoretical_loss": 3.680905553683083, + "tokens_seen": 913768448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003652156469408225, + "loss": 2.7815, + "theoretical_loss": 3.680880010651016, + "tokens_seen": 913833984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036520561685055166, + "loss": 2.8346, + "theoretical_loss": 3.6808544699635837, + "tokens_seen": 913899520 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036519558676028084, + "loss": 2.5171, + "theoretical_loss": 3.680828931620402, + "tokens_seen": 913965056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036518555667001, + "loss": 2.6756, + "theoretical_loss": 3.680803395621088, + "tokens_seen": 914030592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036517552657973926, + "loss": 2.5853, + "theoretical_loss": 3.680777861965259, + "tokens_seen": 914096128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651654964894684, + "loss": 2.5823, + "theoretical_loss": 3.680752330652531, + "tokens_seen": 914161664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 511255, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8368189334869385, + "objective/train/theoretical_loss": 3.680726801682522, + "objective/train/tokens_used": 934687200, + "theoretical_loss": 3.680726801682522, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651554663991976, + "loss": 2.7186, + "theoretical_loss": 3.680726801682522, + "tokens_seen": 914227200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036514543630892675, + "loss": 2.6055, + "theoretical_loss": 3.6807012750548482, + "tokens_seen": 914292736 + }, + { + "epoch": 0.28, + "learning_rate": 0.000365135406218656, + "loss": 2.6287, + "theoretical_loss": 3.6806757507691277, + "tokens_seen": 914358272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036512537612838516, + "loss": 2.552, + "theoretical_loss": 3.6806502288249767, + "tokens_seen": 914423808 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036511534603811434, + "loss": 2.6986, + "theoretical_loss": 3.6806247092220143, + "tokens_seen": 914489344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003651053159478435, + "loss": 2.8004, + "theoretical_loss": 3.6805991919598564, + "tokens_seen": 914554880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036509528585757276, + "loss": 2.5273, + "theoretical_loss": 3.680573677038121, + "tokens_seen": 914620416 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650852557673019, + "loss": 2.7229, + "theoretical_loss": 3.680548164456426, + "tokens_seen": 914685952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650752256770311, + "loss": 2.4541, + "theoretical_loss": 3.680522654214389, + "tokens_seen": 914751488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036506519558676025, + "loss": 2.6284, + "theoretical_loss": 3.680497146311628, + "tokens_seen": 914817024 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650551654964895, + "loss": 2.7057, + "theoretical_loss": 3.6804716407477605, + "tokens_seen": 914882560 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036504513540621867, + "loss": 2.8252, + "theoretical_loss": 3.6804461375224053, + "tokens_seen": 914948096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036503510531594785, + "loss": 2.7718, + "theoretical_loss": 3.68042063663518, + "tokens_seen": 915013632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036502507522567703, + "loss": 2.7386, + "theoretical_loss": 3.6803951380857027, + "tokens_seen": 915079168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650150451354062, + "loss": 2.4951, + "theoretical_loss": 3.6803696418735914, + "tokens_seen": 915144704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003650050150451354, + "loss": 2.8558, + "theoretical_loss": 3.6803441479984653, + "tokens_seen": 915210240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649949849548646, + "loss": 2.743, + "theoretical_loss": 3.6803186564599426, + "tokens_seen": 915275776 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649949849548646, + "loss": 2.5659, + "theoretical_loss": 3.6802931672576413, + "tokens_seen": 915341312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036498495486459375, + "loss": 2.7712, + "theoretical_loss": 3.680267680391181, + "tokens_seen": 915406848 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364974924774323, + "loss": 2.741, + "theoretical_loss": 3.6802421958601794, + "tokens_seen": 915472384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649648946840521, + "loss": 2.5163, + "theoretical_loss": 3.6802167136642563, + "tokens_seen": 915537920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036495486459378135, + "loss": 2.8221, + "theoretical_loss": 3.68019123380303, + "tokens_seen": 915603456 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036494483450351053, + "loss": 2.6905, + "theoretical_loss": 3.6801657562761196, + "tokens_seen": 915668992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649348044132397, + "loss": 2.8177, + "theoretical_loss": 3.6801402810831445, + "tokens_seen": 915734528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003649247743229689, + "loss": 2.8277, + "theoretical_loss": 3.6801148082237236, + "tokens_seen": 915800064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 511864, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.596713066101074, + "objective/train/theoretical_loss": 3.680089337697476, + "objective/train/tokens_used": 936325600, + "theoretical_loss": 3.680089337697476, + "tokens_seen": 915865600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036491474423269813, + "loss": 2.5618, + "theoretical_loss": 3.680089337697476, + "tokens_seen": 915865600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036490471414242726, + "loss": 2.7874, + "theoretical_loss": 3.6800638695040213, + "tokens_seen": 915931136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648946840521565, + "loss": 2.8756, + "theoretical_loss": 3.680038403642979, + "tokens_seen": 915996672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648846539618856, + "loss": 2.9633, + "theoretical_loss": 3.6800129401139685, + "tokens_seen": 916062208 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036487462387161485, + "loss": 2.4687, + "theoretical_loss": 3.67998747891661, + "tokens_seen": 916127744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648645937813441, + "loss": 2.6901, + "theoretical_loss": 3.6799620200505223, + "tokens_seen": 916193280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648545636910732, + "loss": 2.4379, + "theoretical_loss": 3.6799365635153256, + "tokens_seen": 916258816 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036484453360080245, + "loss": 2.813, + "theoretical_loss": 3.6799111093106402, + "tokens_seen": 916324352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648345035105316, + "loss": 2.7068, + "theoretical_loss": 3.6798856574360856, + "tokens_seen": 916389888 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648244734202608, + "loss": 2.9076, + "theoretical_loss": 3.6798602078912817, + "tokens_seen": 916455424 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036481444332999, + "loss": 2.7046, + "theoretical_loss": 3.679834760675849, + "tokens_seen": 916520960 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003648044132397192, + "loss": 2.6643, + "theoretical_loss": 3.679809315789408, + "tokens_seen": 916586496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036479438314944836, + "loss": 2.9033, + "theoretical_loss": 3.679783873231579, + "tokens_seen": 916652032 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036478435305917754, + "loss": 2.7993, + "theoretical_loss": 3.679758433001982, + "tokens_seen": 916717568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647743229689067, + "loss": 2.7358, + "theoretical_loss": 3.6797329951002373, + "tokens_seen": 916783104 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036476429287863595, + "loss": 2.7013, + "theoretical_loss": 3.6797075595259665, + "tokens_seen": 916848640 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647542627883651, + "loss": 2.4434, + "theoretical_loss": 3.6796821262787898, + "tokens_seen": 916914176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647442326980943, + "loss": 2.5695, + "theoretical_loss": 3.679656695358328, + "tokens_seen": 916979712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647342026078235, + "loss": 2.8252, + "theoretical_loss": 3.679631266764201, + "tokens_seen": 917045248 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003647241725175527, + "loss": 2.8018, + "theoretical_loss": 3.6796058404960315, + "tokens_seen": 917110784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036471414242728186, + "loss": 2.804, + "theoretical_loss": 3.6795804165534394, + "tokens_seen": 917176320 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036470411233701104, + "loss": 2.869, + "theoretical_loss": 3.6795549949360464, + "tokens_seen": 917241856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646940822467402, + "loss": 2.7698, + "theoretical_loss": 3.6795295756434734, + "tokens_seen": 917307392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036468405215646946, + "loss": 2.5769, + "theoretical_loss": 3.679504158675342, + "tokens_seen": 917372928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646740220661986, + "loss": 2.6722, + "theoretical_loss": 3.6794787440312735, + "tokens_seen": 917438464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 513081, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8393032550811768, + "objective/train/theoretical_loss": 3.679453331710889, + "objective/train/tokens_used": 937964000, + "theoretical_loss": 3.679453331710889, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646639919759278, + "loss": 2.6943, + "theoretical_loss": 3.679453331710889, + "tokens_seen": 917504000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036465396188565695, + "loss": 3.0041, + "theoretical_loss": 3.6794279217138106, + "tokens_seen": 917569536 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646439317953862, + "loss": 2.7193, + "theoretical_loss": 3.6794025140396602, + "tokens_seen": 917635072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036463390170511536, + "loss": 2.8523, + "theoretical_loss": 3.6793771086880587, + "tokens_seen": 917700608 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036462387161484454, + "loss": 2.6422, + "theoretical_loss": 3.6793517056586285, + "tokens_seen": 917766144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003646138415245737, + "loss": 2.8937, + "theoretical_loss": 3.6793263049509917, + "tokens_seen": 917831680 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036460381143430296, + "loss": 2.6182, + "theoretical_loss": 3.67930090656477, + "tokens_seen": 917897216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645937813440321, + "loss": 2.6326, + "theoretical_loss": 3.6792755104995853, + "tokens_seen": 917962752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645837512537613, + "loss": 2.6397, + "theoretical_loss": 3.6792501167550604, + "tokens_seen": 918028288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036457372116349045, + "loss": 2.5801, + "theoretical_loss": 3.679224725330818, + "tokens_seen": 918093824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645636910732197, + "loss": 2.7532, + "theoretical_loss": 3.679199336226479, + "tokens_seen": 918159360 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036455366098294887, + "loss": 2.7627, + "theoretical_loss": 3.679173949441667, + "tokens_seen": 918224896 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036454363089267805, + "loss": 2.7622, + "theoretical_loss": 3.6791485649760043, + "tokens_seen": 918290432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036453360080240723, + "loss": 2.7045, + "theoretical_loss": 3.6791231828291133, + "tokens_seen": 918355968 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645235707121364, + "loss": 2.7075, + "theoretical_loss": 3.679097803000617, + "tokens_seen": 918421504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645135406218656, + "loss": 2.8868, + "theoretical_loss": 3.6790724254901384, + "tokens_seen": 918487040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003645035105315948, + "loss": 2.8046, + "theoretical_loss": 3.6790470502973, + "tokens_seen": 918552576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036449348044132395, + "loss": 2.6042, + "theoretical_loss": 3.679021677421725, + "tokens_seen": 918618112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644834503510532, + "loss": 2.8111, + "theoretical_loss": 3.678996306863037, + "tokens_seen": 918683648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644734202607823, + "loss": 2.713, + "theoretical_loss": 3.678970938620858, + "tokens_seen": 918749184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036446339017051155, + "loss": 2.6038, + "theoretical_loss": 3.678945572694812, + "tokens_seen": 918814720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036445336008024073, + "loss": 2.6464, + "theoretical_loss": 3.6789202090845223, + "tokens_seen": 918880256 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644433299899699, + "loss": 2.7903, + "theoretical_loss": 3.6788948477896124, + "tokens_seen": 918945792 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644332998996991, + "loss": 2.7264, + "theoretical_loss": 3.678869488809706, + "tokens_seen": 919011328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036442326980942833, + "loss": 2.5116, + "theoretical_loss": 3.6788441321444267, + "tokens_seen": 919076864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 514023, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6861374378204346, + "objective/train/theoretical_loss": 3.6788187777933974, + "objective/train/tokens_used": 939602400, + "theoretical_loss": 3.6788187777933974, + "tokens_seen": 919142400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036441323971915746, + "loss": 2.5727, + "theoretical_loss": 3.6788187777933974, + "tokens_seen": 919142400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003644032096288867, + "loss": 2.7196, + "theoretical_loss": 3.678793425756243, + "tokens_seen": 919207936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643931795386158, + "loss": 2.6527, + "theoretical_loss": 3.6787680760325863, + "tokens_seen": 919273472 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036438314944834505, + "loss": 2.7031, + "theoretical_loss": 3.678742728622052, + "tokens_seen": 919339008 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036437311935807424, + "loss": 2.7691, + "theoretical_loss": 3.6787173835242646, + "tokens_seen": 919404544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643630892678034, + "loss": 2.6177, + "theoretical_loss": 3.6786920407388473, + "tokens_seen": 919470080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643530591775326, + "loss": 2.7784, + "theoretical_loss": 3.678666700265425, + "tokens_seen": 919535616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643430290872618, + "loss": 2.586, + "theoretical_loss": 3.678641362103621, + "tokens_seen": 919601152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036433299899699096, + "loss": 2.5905, + "theoretical_loss": 3.678616026253061, + "tokens_seen": 919666688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643229689067202, + "loss": 2.6251, + "theoretical_loss": 3.6785906927133687, + "tokens_seen": 919732224 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003643129388164493, + "loss": 2.748, + "theoretical_loss": 3.678565361484169, + "tokens_seen": 919797760 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036430290872617856, + "loss": 2.8521, + "theoretical_loss": 3.6785400325650865, + "tokens_seen": 919863296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642928786359077, + "loss": 2.71, + "theoretical_loss": 3.678514705955746, + "tokens_seen": 919928832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642828485456369, + "loss": 2.6943, + "theoretical_loss": 3.678489381655772, + "tokens_seen": 919994368 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642728184553661, + "loss": 2.9214, + "theoretical_loss": 3.6784640596647904, + "tokens_seen": 920059904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642627883650953, + "loss": 2.8038, + "theoretical_loss": 3.6784387399824254, + "tokens_seen": 920125440 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036425275827482446, + "loss": 2.9118, + "theoretical_loss": 3.678413422608302, + "tokens_seen": 920190976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642427281845537, + "loss": 2.4233, + "theoretical_loss": 3.6783881075420455, + "tokens_seen": 920256512 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642326980942828, + "loss": 2.7865, + "theoretical_loss": 3.6783627947832818, + "tokens_seen": 920322048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036422266800401206, + "loss": 2.6493, + "theoretical_loss": 3.678337484331636, + "tokens_seen": 920387584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642126379137412, + "loss": 2.8862, + "theoretical_loss": 3.6783121761867332, + "tokens_seen": 920453120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003642026078234704, + "loss": 2.6877, + "theoretical_loss": 3.678286870348199, + "tokens_seen": 920518656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641925777331996, + "loss": 2.7539, + "theoretical_loss": 3.6782615668156593, + "tokens_seen": 920584192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641825476429288, + "loss": 2.793, + "theoretical_loss": 3.67823626558874, + "tokens_seen": 920649728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036417251755265797, + "loss": 2.5625, + "theoretical_loss": 3.6782109666670664, + "tokens_seen": 920715264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 514680, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3389620780944824, + "objective/train/theoretical_loss": 3.6781856700502646, + "objective/train/tokens_used": 941240800, + "theoretical_loss": 3.6781856700502646, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036416248746238715, + "loss": 2.6741, + "theoretical_loss": 3.6781856700502646, + "tokens_seen": 920780800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036415245737211633, + "loss": 2.5103, + "theoretical_loss": 3.678160375737961, + "tokens_seen": 920846336 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036414242728184556, + "loss": 2.7652, + "theoretical_loss": 3.678135083729781, + "tokens_seen": 920911872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641323971915747, + "loss": 2.6444, + "theoretical_loss": 3.6781097940253513, + "tokens_seen": 920977408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641223671013039, + "loss": 2.6945, + "theoretical_loss": 3.6780845066242978, + "tokens_seen": 921042944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036411233701103316, + "loss": 2.9332, + "theoretical_loss": 3.678059221526247, + "tokens_seen": 921108480 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003641023069207623, + "loss": 2.614, + "theoretical_loss": 3.678033938730825, + "tokens_seen": 921174016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640922768304915, + "loss": 2.7244, + "theoretical_loss": 3.6780086582376588, + "tokens_seen": 921239552 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036408224674022065, + "loss": 2.7352, + "theoretical_loss": 3.677983380046375, + "tokens_seen": 921305088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640722166499499, + "loss": 2.4935, + "theoretical_loss": 3.6779581041566, + "tokens_seen": 921370624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036406218655967907, + "loss": 2.4393, + "theoretical_loss": 3.6779328305679604, + "tokens_seen": 921436160 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036405215646940825, + "loss": 2.6316, + "theoretical_loss": 3.6779075592800834, + "tokens_seen": 921501696 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036404212637913743, + "loss": 2.7714, + "theoretical_loss": 3.677882290292596, + "tokens_seen": 921567232 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640320962888666, + "loss": 2.6272, + "theoretical_loss": 3.677857023605125, + "tokens_seen": 921632768 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003640220661985958, + "loss": 2.7397, + "theoretical_loss": 3.6778317592172973, + "tokens_seen": 921698304 + }, + { + "epoch": 0.28, + "learning_rate": 0.000364012036108325, + "loss": 2.6956, + "theoretical_loss": 3.677806497128741, + "tokens_seen": 921763840 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036400200601805415, + "loss": 2.8998, + "theoretical_loss": 3.677781237339083, + "tokens_seen": 921829376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639919759277834, + "loss": 2.5165, + "theoretical_loss": 3.6777559798479498, + "tokens_seen": 921894912 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639819458375125, + "loss": 2.685, + "theoretical_loss": 3.67773072465497, + "tokens_seen": 921960448 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036397191574724175, + "loss": 2.6519, + "theoretical_loss": 3.677705471759771, + "tokens_seen": 922025984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036396188565697093, + "loss": 2.6, + "theoretical_loss": 3.6776802211619795, + "tokens_seen": 922091520 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639518555667001, + "loss": 2.7074, + "theoretical_loss": 3.677654972861224, + "tokens_seen": 922157056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639418254764293, + "loss": 2.7473, + "theoretical_loss": 3.6776297268571323, + "tokens_seen": 922222592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036393179538615853, + "loss": 2.8253, + "theoretical_loss": 3.6776044831493326, + "tokens_seen": 922288128 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036392176529588766, + "loss": 2.7029, + "theoretical_loss": 3.677579241737452, + "tokens_seen": 922353664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 515999, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7671639919281006, + "objective/train/theoretical_loss": 3.677554002621119, + "objective/train/tokens_used": 942879200, + "theoretical_loss": 3.677554002621119, + "tokens_seen": 922419200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003639117352056169, + "loss": 2.6393, + "theoretical_loss": 3.677554002621119, + "tokens_seen": 922419200 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363901705115346, + "loss": 2.7322, + "theoretical_loss": 3.6775287657999627, + "tokens_seen": 922484736 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036389167502507525, + "loss": 2.7465, + "theoretical_loss": 3.6775035312736097, + "tokens_seen": 922550272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036388164493480444, + "loss": 2.7354, + "theoretical_loss": 3.67747829904169, + "tokens_seen": 922615808 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638716148445336, + "loss": 2.7393, + "theoretical_loss": 3.67745306910383, + "tokens_seen": 922681344 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638615847542628, + "loss": 2.6674, + "theoretical_loss": 3.67742784145966, + "tokens_seen": 922746880 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363851554663992, + "loss": 2.9504, + "theoretical_loss": 3.6774026161088083, + "tokens_seen": 922812416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036384152457372116, + "loss": 2.645, + "theoretical_loss": 3.6773773930509024, + "tokens_seen": 922877952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638314944834504, + "loss": 2.6187, + "theoretical_loss": 3.6773521722855724, + "tokens_seen": 922943488 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638214643931795, + "loss": 2.6137, + "theoretical_loss": 3.6773269538124467, + "tokens_seen": 923009024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036381143430290876, + "loss": 2.5773, + "theoretical_loss": 3.677301737631154, + "tokens_seen": 923074560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003638014042126379, + "loss": 2.6507, + "theoretical_loss": 3.677276523741323, + "tokens_seen": 923140096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637913741223671, + "loss": 2.8147, + "theoretical_loss": 3.6772513121425847, + "tokens_seen": 923205632 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637813440320963, + "loss": 2.9722, + "theoretical_loss": 3.677226102834566, + "tokens_seen": 923271168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637713139418255, + "loss": 2.8759, + "theoretical_loss": 3.677200895816897, + "tokens_seen": 923336704 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036376128385155466, + "loss": 2.895, + "theoretical_loss": 3.6771756910892077, + "tokens_seen": 923402240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637512537612839, + "loss": 2.7822, + "theoretical_loss": 3.677150488651127, + "tokens_seen": 923467776 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363741223671013, + "loss": 2.752, + "theoretical_loss": 3.6771252885022845, + "tokens_seen": 923533312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036373119358074226, + "loss": 2.7301, + "theoretical_loss": 3.6771000906423095, + "tokens_seen": 923598848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637211634904714, + "loss": 2.8236, + "theoretical_loss": 3.6770748950708323, + "tokens_seen": 923664384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637111334002006, + "loss": 2.8326, + "theoretical_loss": 3.6770497017874826, + "tokens_seen": 923729920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003637011033099298, + "loss": 2.6637, + "theoretical_loss": 3.67702451079189, + "tokens_seen": 923795456 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363691073219659, + "loss": 2.5562, + "theoretical_loss": 3.6769993220836845, + "tokens_seen": 923860992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036368104312938817, + "loss": 2.7717, + "theoretical_loss": 3.6769741356624968, + "tokens_seen": 923926528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036367101303911735, + "loss": 2.9733, + "theoretical_loss": 3.676948951527956, + "tokens_seen": 923992064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 516502, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.229719638824463, + "objective/train/theoretical_loss": 3.6769237696796933, + "objective/train/tokens_used": 944517600, + "theoretical_loss": 3.6769237696796933, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036366098294884653, + "loss": 2.9243, + "theoretical_loss": 3.6769237696796933, + "tokens_seen": 924057600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036365095285857576, + "loss": 2.5783, + "theoretical_loss": 3.676898590117338, + "tokens_seen": 924123136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636409227683049, + "loss": 2.8002, + "theoretical_loss": 3.6768734128405214, + "tokens_seen": 924188672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636308926780341, + "loss": 2.5052, + "theoretical_loss": 3.6768482378488745, + "tokens_seen": 924254208 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036362086258776325, + "loss": 2.8707, + "theoretical_loss": 3.6768230651420257, + "tokens_seen": 924319744 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003636108324974925, + "loss": 2.6332, + "theoretical_loss": 3.6767978947196083, + "tokens_seen": 924385280 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036360080240722167, + "loss": 2.8033, + "theoretical_loss": 3.676772726581251, + "tokens_seen": 924450816 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036359077231695085, + "loss": 2.6453, + "theoretical_loss": 3.676747560726586, + "tokens_seen": 924516352 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036358074222668003, + "loss": 2.7782, + "theoretical_loss": 3.6767223971552427, + "tokens_seen": 924581888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036357071213640927, + "loss": 2.6504, + "theoretical_loss": 3.676697235866854, + "tokens_seen": 924647424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003635606820461384, + "loss": 2.8365, + "theoretical_loss": 3.67667207686105, + "tokens_seen": 924712960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036355065195586763, + "loss": 2.769, + "theoretical_loss": 3.676646920137461, + "tokens_seen": 924778496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036354062186559676, + "loss": 2.5838, + "theoretical_loss": 3.67662176569572, + "tokens_seen": 924844032 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363530591775326, + "loss": 2.4492, + "theoretical_loss": 3.6765966135354575, + "tokens_seen": 924909568 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036352056168505517, + "loss": 2.735, + "theoretical_loss": 3.676571463656305, + "tokens_seen": 924975104 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036351053159478435, + "loss": 2.6674, + "theoretical_loss": 3.6765463160578937, + "tokens_seen": 925040640 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036350050150451353, + "loss": 2.4657, + "theoretical_loss": 3.676521170739856, + "tokens_seen": 925106176 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634904714142427, + "loss": 2.6863, + "theoretical_loss": 3.676496027701823, + "tokens_seen": 925171712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634804413239719, + "loss": 2.806, + "theoretical_loss": 3.6764708869434264, + "tokens_seen": 925237248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036347041123370113, + "loss": 2.6572, + "theoretical_loss": 3.676445748464298, + "tokens_seen": 925302784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036346038114343026, + "loss": 2.612, + "theoretical_loss": 3.67642061226407, + "tokens_seen": 925368320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634503510531595, + "loss": 2.6244, + "theoretical_loss": 3.676395478342375, + "tokens_seen": 925433856 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634403209628886, + "loss": 2.7324, + "theoretical_loss": 3.676370346698844, + "tokens_seen": 925499392 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036343029087261786, + "loss": 2.6495, + "theoretical_loss": 3.67634521733311, + "tokens_seen": 925564928 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036342026078234704, + "loss": 2.6261, + "theoretical_loss": 3.6763200902448054, + "tokens_seen": 925630464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 517571, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.828040361404419, + "objective/train/theoretical_loss": 3.676294965433562, + "objective/train/tokens_used": 946156000, + "theoretical_loss": 3.676294965433562, + "tokens_seen": 925696000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634102306920762, + "loss": 2.7145, + "theoretical_loss": 3.676294965433562, + "tokens_seen": 925696000 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003634002006018054, + "loss": 2.8367, + "theoretical_loss": 3.6762698428990124, + "tokens_seen": 925761536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036339017051153464, + "loss": 2.6012, + "theoretical_loss": 3.6762447226407895, + "tokens_seen": 925827072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036338014042126376, + "loss": 2.668, + "theoretical_loss": 3.6762196046585256, + "tokens_seen": 925892608 + }, + { + "epoch": 0.28, + "learning_rate": 0.000363370110330993, + "loss": 2.8896, + "theoretical_loss": 3.676194488951853, + "tokens_seen": 925958144 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633600802407222, + "loss": 2.8035, + "theoretical_loss": 3.6761693755204057, + "tokens_seen": 926023680 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036335005015045136, + "loss": 2.8345, + "theoretical_loss": 3.676144264363815, + "tokens_seen": 926089216 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633400200601806, + "loss": 2.8302, + "theoretical_loss": 3.6761191554817154, + "tokens_seen": 926154752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633299899699097, + "loss": 2.5149, + "theoretical_loss": 3.6760940488737397, + "tokens_seen": 926220288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036331995987963896, + "loss": 2.4549, + "theoretical_loss": 3.6760689445395203, + "tokens_seen": 926285824 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003633099297893681, + "loss": 2.7747, + "theoretical_loss": 3.6760438424786903, + "tokens_seen": 926351360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632998996990973, + "loss": 2.7876, + "theoretical_loss": 3.676018742690884, + "tokens_seen": 926416896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632898696088265, + "loss": 2.7019, + "theoretical_loss": 3.6759936451757342, + "tokens_seen": 926482432 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632798395185557, + "loss": 2.8405, + "theoretical_loss": 3.675968549932875, + "tokens_seen": 926547968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036326980942828486, + "loss": 2.6827, + "theoretical_loss": 3.6759434569619387, + "tokens_seen": 926613504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632597793380141, + "loss": 2.4907, + "theoretical_loss": 3.67591836626256, + "tokens_seen": 926679040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632497492477432, + "loss": 2.8732, + "theoretical_loss": 3.6758932778343727, + "tokens_seen": 926744576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036323971915747246, + "loss": 2.6314, + "theoretical_loss": 3.6758681916770097, + "tokens_seen": 926810112 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632296890672016, + "loss": 2.7107, + "theoretical_loss": 3.675843107790106, + "tokens_seen": 926875648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003632196589769308, + "loss": 2.7786, + "theoretical_loss": 3.675818026173295, + "tokens_seen": 926941184 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036320962888666, + "loss": 2.6493, + "theoretical_loss": 3.6757929468262107, + "tokens_seen": 927006720 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631995987963892, + "loss": 2.5168, + "theoretical_loss": 3.6757678697484875, + "tokens_seen": 927072256 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036318956870611837, + "loss": 2.7334, + "theoretical_loss": 3.67574279493976, + "tokens_seen": 927137792 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036317953861584755, + "loss": 2.6639, + "theoretical_loss": 3.6757177223996615, + "tokens_seen": 927203328 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036316950852557673, + "loss": 2.7426, + "theoretical_loss": 3.675692652127827, + "tokens_seen": 927268864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 518862, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0217199325561523, + "objective/train/theoretical_loss": 3.6756675841238913, + "objective/train/tokens_used": 947794400, + "theoretical_loss": 3.6756675841238913, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036315947843530596, + "loss": 2.8993, + "theoretical_loss": 3.6756675841238913, + "tokens_seen": 927334400 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631494483450351, + "loss": 2.385, + "theoretical_loss": 3.6756425183874892, + "tokens_seen": 927399936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631394182547643, + "loss": 2.7425, + "theoretical_loss": 3.675617454918254, + "tokens_seen": 927465472 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036312938816449345, + "loss": 2.7258, + "theoretical_loss": 3.675592393715822, + "tokens_seen": 927531008 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003631193580742227, + "loss": 2.6855, + "theoretical_loss": 3.675567334779827, + "tokens_seen": 927596544 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036310932798395187, + "loss": 3.0311, + "theoretical_loss": 3.675542278109905, + "tokens_seen": 927662080 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036309929789368105, + "loss": 2.6216, + "theoretical_loss": 3.6755172237056897, + "tokens_seen": 927727616 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036308926780341023, + "loss": 2.2776, + "theoretical_loss": 3.675492171566817, + "tokens_seen": 927793152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036307923771313947, + "loss": 2.6105, + "theoretical_loss": 3.675467121692922, + "tokens_seen": 927858688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630692076228686, + "loss": 2.8852, + "theoretical_loss": 3.67544207408364, + "tokens_seen": 927924224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036305917753259783, + "loss": 2.6106, + "theoretical_loss": 3.675417028738606, + "tokens_seen": 927989760 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036304914744232696, + "loss": 2.7, + "theoretical_loss": 3.675391985657456, + "tokens_seen": 928055296 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003630391173520562, + "loss": 2.6264, + "theoretical_loss": 3.675366944839825, + "tokens_seen": 928120832 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036302908726178537, + "loss": 2.6397, + "theoretical_loss": 3.6753419062853485, + "tokens_seen": 928186368 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036301905717151455, + "loss": 2.5242, + "theoretical_loss": 3.6753168699936625, + "tokens_seen": 928251904 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036300902708124374, + "loss": 2.6, + "theoretical_loss": 3.675291835964403, + "tokens_seen": 928317440 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629989969909729, + "loss": 2.7545, + "theoretical_loss": 3.675266804197206, + "tokens_seen": 928382976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629889669007021, + "loss": 2.8854, + "theoretical_loss": 3.6752417746917065, + "tokens_seen": 928448512 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036297893681043133, + "loss": 2.6897, + "theoretical_loss": 3.675216747447541, + "tokens_seen": 928514048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036296890672016046, + "loss": 2.7338, + "theoretical_loss": 3.675191722464346, + "tokens_seen": 928579584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629588766298897, + "loss": 2.6159, + "theoretical_loss": 3.6751666997417574, + "tokens_seen": 928645120 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629488465396188, + "loss": 2.8166, + "theoretical_loss": 3.6751416792794114, + "tokens_seen": 928710656 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036293881644934806, + "loss": 2.6078, + "theoretical_loss": 3.6751166610769443, + "tokens_seen": 928776192 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036292878635907724, + "loss": 2.8892, + "theoretical_loss": 3.6750916451339926, + "tokens_seen": 928841728 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629187562688064, + "loss": 2.4984, + "theoretical_loss": 3.6750666314501927, + "tokens_seen": 928907264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 519491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.917207717895508, + "objective/train/theoretical_loss": 3.675041620025182, + "objective/train/tokens_used": 949432800, + "theoretical_loss": 3.675041620025182, + "tokens_seen": 928972800 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003629087261785356, + "loss": 2.6886, + "theoretical_loss": 3.675041620025182, + "tokens_seen": 928972800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036289869608826484, + "loss": 2.6166, + "theoretical_loss": 3.675016610858596, + "tokens_seen": 929038336 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036288866599799396, + "loss": 2.7116, + "theoretical_loss": 3.6749916039500725, + "tokens_seen": 929103872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628786359077232, + "loss": 2.5936, + "theoretical_loss": 3.674966599299248, + "tokens_seen": 929169408 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628686058174523, + "loss": 2.6567, + "theoretical_loss": 3.674941596905759, + "tokens_seen": 929234944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036285857572718156, + "loss": 2.7593, + "theoretical_loss": 3.6749165967692434, + "tokens_seen": 929300480 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036284854563691074, + "loss": 2.6485, + "theoretical_loss": 3.674891598889338, + "tokens_seen": 929366016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628385155466399, + "loss": 2.7085, + "theoretical_loss": 3.674866603265679, + "tokens_seen": 929431552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628284854563691, + "loss": 2.5491, + "theoretical_loss": 3.674841609897905, + "tokens_seen": 929497088 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003628184553660983, + "loss": 2.6976, + "theoretical_loss": 3.674816618785653, + "tokens_seen": 929562624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036280842527582747, + "loss": 2.5553, + "theoretical_loss": 3.67479162992856, + "tokens_seen": 929628160 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627983951855567, + "loss": 2.8656, + "theoretical_loss": 3.6747666433262642, + "tokens_seen": 929693696 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036278836509528583, + "loss": 2.6386, + "theoretical_loss": 3.674741658978403, + "tokens_seen": 929759232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036277833500501506, + "loss": 2.6119, + "theoretical_loss": 3.6747166768846142, + "tokens_seen": 929824768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036276830491474424, + "loss": 2.6403, + "theoretical_loss": 3.674691697044535, + "tokens_seen": 929890304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627582748244734, + "loss": 2.7474, + "theoretical_loss": 3.6746667194578038, + "tokens_seen": 929955840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627482447342026, + "loss": 2.6571, + "theoretical_loss": 3.6746417441240586, + "tokens_seen": 930021376 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627382146439318, + "loss": 2.82, + "theoretical_loss": 3.674616771042937, + "tokens_seen": 930086912 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036272818455366097, + "loss": 2.8842, + "theoretical_loss": 3.6745918002140776, + "tokens_seen": 930152448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003627181544633902, + "loss": 2.7982, + "theoretical_loss": 3.6745668316371183, + "tokens_seen": 930217984 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036270812437311933, + "loss": 2.6886, + "theoretical_loss": 3.6745418653116975, + "tokens_seen": 930283520 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036269809428284857, + "loss": 2.7248, + "theoretical_loss": 3.6745169012374537, + "tokens_seen": 930349056 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626880641925777, + "loss": 2.6175, + "theoretical_loss": 3.6744919394140254, + "tokens_seen": 930414592 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036267803410230693, + "loss": 2.8798, + "theoretical_loss": 3.6744669798410508, + "tokens_seen": 930480128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626680040120361, + "loss": 2.7445, + "theoretical_loss": 3.6744420225181686, + "tokens_seen": 930545664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 520743, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.064671277999878, + "objective/train/theoretical_loss": 3.6744170674450176, + "objective/train/tokens_used": 951071200, + "theoretical_loss": 3.6744170674450176, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003626579739217653, + "loss": 2.841, + "theoretical_loss": 3.6744170674450176, + "tokens_seen": 930611200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036264794383149447, + "loss": 2.5725, + "theoretical_loss": 3.6743921146212366, + "tokens_seen": 930676736 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036263791374122365, + "loss": 2.7547, + "theoretical_loss": 3.6743671640464646, + "tokens_seen": 930742272 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036262788365095283, + "loss": 2.7484, + "theoretical_loss": 3.6743422157203405, + "tokens_seen": 930807808 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036261785356068207, + "loss": 2.5509, + "theoretical_loss": 3.6743172696425033, + "tokens_seen": 930873344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036260782347041125, + "loss": 2.5975, + "theoretical_loss": 3.6742923258125915, + "tokens_seen": 930938880 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036259779338014043, + "loss": 2.8421, + "theoretical_loss": 3.674267384230246, + "tokens_seen": 931004416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036258776328986967, + "loss": 2.928, + "theoretical_loss": 3.6742424448951043, + "tokens_seen": 931069952 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625777331995988, + "loss": 2.7928, + "theoretical_loss": 3.6742175078068064, + "tokens_seen": 931135488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036256770310932803, + "loss": 2.6882, + "theoretical_loss": 3.674192572964992, + "tokens_seen": 931201024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036255767301905716, + "loss": 2.8744, + "theoretical_loss": 3.6741676403693004, + "tokens_seen": 931266560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625476429287864, + "loss": 2.51, + "theoretical_loss": 3.6741427100193715, + "tokens_seen": 931332096 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625376128385156, + "loss": 2.8063, + "theoretical_loss": 3.6741177819148447, + "tokens_seen": 931397632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036252758274824475, + "loss": 2.8343, + "theoretical_loss": 3.67409285605536, + "tokens_seen": 931463168 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036251755265797394, + "loss": 2.771, + "theoretical_loss": 3.674067932440557, + "tokens_seen": 931528704 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003625075225677031, + "loss": 2.5716, + "theoretical_loss": 3.674043011070076, + "tokens_seen": 931594240 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624974924774323, + "loss": 2.7609, + "theoretical_loss": 3.674018091943557, + "tokens_seen": 931659776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036248746238716153, + "loss": 2.7397, + "theoretical_loss": 3.67399317506064, + "tokens_seen": 931725312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036247743229689066, + "loss": 2.6042, + "theoretical_loss": 3.673968260420965, + "tokens_seen": 931790848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624674022066199, + "loss": 2.5878, + "theoretical_loss": 3.6739433480241726, + "tokens_seen": 931856384 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362457372116349, + "loss": 2.8429, + "theoretical_loss": 3.6739184378699026, + "tokens_seen": 931921920 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036244734202607826, + "loss": 2.447, + "theoretical_loss": 3.6738935299577964, + "tokens_seen": 931987456 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036243731193580744, + "loss": 2.7397, + "theoretical_loss": 3.6738686242874943, + "tokens_seen": 932052992 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624272818455366, + "loss": 2.5438, + "theoretical_loss": 3.6738437208586365, + "tokens_seen": 932118528 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003624172517552658, + "loss": 2.8547, + "theoretical_loss": 3.6738188196708634, + "tokens_seen": 932184064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 521434, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9683523178100586, + "objective/train/theoretical_loss": 3.673793920723817, + "objective/train/tokens_used": 952709600, + "theoretical_loss": 3.673793920723817, + "tokens_seen": 932249600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036240722166499504, + "loss": 2.8287, + "theoretical_loss": 3.673793920723817, + "tokens_seen": 932249600 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036239719157472416, + "loss": 2.4303, + "theoretical_loss": 3.673769024017137, + "tokens_seen": 932315136 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623871614844534, + "loss": 2.56, + "theoretical_loss": 3.6737441295504647, + "tokens_seen": 932380672 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623771313941825, + "loss": 2.6744, + "theoretical_loss": 3.6737192373234415, + "tokens_seen": 932446208 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036236710130391176, + "loss": 2.8192, + "theoretical_loss": 3.6736943473357084, + "tokens_seen": 932511744 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036235707121364094, + "loss": 2.8758, + "theoretical_loss": 3.673669459586906, + "tokens_seen": 932577280 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623470411233701, + "loss": 2.6161, + "theoretical_loss": 3.6736445740766763, + "tokens_seen": 932642816 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623370110330993, + "loss": 2.5468, + "theoretical_loss": 3.673619690804661, + "tokens_seen": 932708352 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623269809428285, + "loss": 2.8673, + "theoretical_loss": 3.6735948097705, + "tokens_seen": 932773888 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036231695085255767, + "loss": 2.694, + "theoretical_loss": 3.6735699309738363, + "tokens_seen": 932839424 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003623069207622869, + "loss": 2.5771, + "theoretical_loss": 3.6735450544143107, + "tokens_seen": 932904960 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036229689067201603, + "loss": 2.638, + "theoretical_loss": 3.6735201800915656, + "tokens_seen": 932970496 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036228686058174526, + "loss": 2.8063, + "theoretical_loss": 3.6734953080052417, + "tokens_seen": 933036032 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036227683049147444, + "loss": 2.7244, + "theoretical_loss": 3.673470438154982, + "tokens_seen": 933101568 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622668004012036, + "loss": 2.8428, + "theoretical_loss": 3.6734455705404283, + "tokens_seen": 933167104 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622567703109328, + "loss": 2.7789, + "theoretical_loss": 3.6734207051612215, + "tokens_seen": 933232640 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362246740220662, + "loss": 2.8115, + "theoretical_loss": 3.673395842017005, + "tokens_seen": 933298176 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036223671013039117, + "loss": 2.5103, + "theoretical_loss": 3.6733709811074204, + "tokens_seen": 933363712 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003622266800401204, + "loss": 2.604, + "theoretical_loss": 3.6733461224321102, + "tokens_seen": 933429248 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036221664994984953, + "loss": 2.9288, + "theoretical_loss": 3.673321265990717, + "tokens_seen": 933494784 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036220661985957877, + "loss": 2.6005, + "theoretical_loss": 3.673296411782882, + "tokens_seen": 933560320 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621965897693079, + "loss": 2.5925, + "theoretical_loss": 3.6732715598082493, + "tokens_seen": 933625856 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036218655967903713, + "loss": 2.6181, + "theoretical_loss": 3.6732467100664605, + "tokens_seen": 933691392 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621765295887663, + "loss": 2.6131, + "theoretical_loss": 3.673221862557159, + "tokens_seen": 933756928 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621664994984955, + "loss": 2.5553, + "theoretical_loss": 3.6731970172799864, + "tokens_seen": 933822464 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 522330, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3445885181427, + "objective/train/theoretical_loss": 3.673172174234587, + "objective/train/tokens_used": 954348000, + "theoretical_loss": 3.673172174234587, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036215646940822467, + "loss": 2.617, + "theoretical_loss": 3.673172174234587, + "tokens_seen": 933888000 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036214643931795385, + "loss": 2.6809, + "theoretical_loss": 3.673147333420603, + "tokens_seen": 933953536 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036213640922768303, + "loss": 2.634, + "theoretical_loss": 3.673122494837677, + "tokens_seen": 934019072 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036212637913741227, + "loss": 2.6592, + "theoretical_loss": 3.6730976584854527, + "tokens_seen": 934084608 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003621163490471414, + "loss": 2.7614, + "theoretical_loss": 3.6730728243635733, + "tokens_seen": 934150144 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036210631895687063, + "loss": 2.7846, + "theoretical_loss": 3.673047992471682, + "tokens_seen": 934215680 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620962888665998, + "loss": 2.5949, + "theoretical_loss": 3.6730231628094216, + "tokens_seen": 934281216 + }, + { + "epoch": 0.28, + "learning_rate": 0.000362086258776329, + "loss": 2.7675, + "theoretical_loss": 3.672998335376436, + "tokens_seen": 934346752 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620762286860582, + "loss": 2.8264, + "theoretical_loss": 3.672973510172369, + "tokens_seen": 934412288 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036206619859578736, + "loss": 2.6534, + "theoretical_loss": 3.6729486871968637, + "tokens_seen": 934477824 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036205616850551654, + "loss": 2.724, + "theoretical_loss": 3.672923866449564, + "tokens_seen": 934543360 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620461384152458, + "loss": 2.7642, + "theoretical_loss": 3.6728990479301133, + "tokens_seen": 934608896 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620361083249749, + "loss": 2.8175, + "theoretical_loss": 3.672874231638156, + "tokens_seen": 934674432 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036202607823470414, + "loss": 2.6981, + "theoretical_loss": 3.672849417573336, + "tokens_seen": 934739968 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036201604814443326, + "loss": 2.4681, + "theoretical_loss": 3.672824605735297, + "tokens_seen": 934805504 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003620060180541625, + "loss": 2.7621, + "theoretical_loss": 3.6727997961236825, + "tokens_seen": 934871040 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619959879638917, + "loss": 2.7309, + "theoretical_loss": 3.6727749887381376, + "tokens_seen": 934936576 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036198595787362086, + "loss": 2.7253, + "theoretical_loss": 3.672750183578306, + "tokens_seen": 935002112 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036197592778335004, + "loss": 2.8102, + "theoretical_loss": 3.6727253806438327, + "tokens_seen": 935067648 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619658976930792, + "loss": 2.5928, + "theoretical_loss": 3.6727005799343613, + "tokens_seen": 935133184 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619558676028084, + "loss": 2.6811, + "theoretical_loss": 3.672675781449537, + "tokens_seen": 935198720 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036194583751253764, + "loss": 2.7066, + "theoretical_loss": 3.6726509851890032, + "tokens_seen": 935264256 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036193580742226677, + "loss": 2.7786, + "theoretical_loss": 3.672626191152406, + "tokens_seen": 935329792 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361925777331996, + "loss": 2.6312, + "theoretical_loss": 3.67260139933939, + "tokens_seen": 935395328 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003619157472417252, + "loss": 2.7573, + "theoretical_loss": 3.6725766097495987, + "tokens_seen": 935460864 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 523607, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.482036590576172, + "objective/train/theoretical_loss": 3.672551822382678, + "objective/train/tokens_used": 955986400, + "theoretical_loss": 3.672551822382678, + "tokens_seen": 935526400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036190571715145436, + "loss": 2.4688, + "theoretical_loss": 3.672551822382678, + "tokens_seen": 935526400 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036189568706118354, + "loss": 2.8175, + "theoretical_loss": 3.672527037238273, + "tokens_seen": 935591936 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618856569709127, + "loss": 2.8405, + "theoretical_loss": 3.6725022543160284, + "tokens_seen": 935657472 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618756268806419, + "loss": 2.688, + "theoretical_loss": 3.672477473615589, + "tokens_seen": 935723008 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036186559679037114, + "loss": 2.5455, + "theoretical_loss": 3.6724526951366006, + "tokens_seen": 935788544 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618555667001003, + "loss": 2.9469, + "theoretical_loss": 3.672427918878708, + "tokens_seen": 935854080 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618455366098295, + "loss": 2.7401, + "theoretical_loss": 3.6724031448415575, + "tokens_seen": 935919616 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618355065195587, + "loss": 2.937, + "theoretical_loss": 3.672378373024794, + "tokens_seen": 935985152 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036182547642928787, + "loss": 2.4816, + "theoretical_loss": 3.672353603428063, + "tokens_seen": 936050688 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003618154463390171, + "loss": 2.674, + "theoretical_loss": 3.6723288360510096, + "tokens_seen": 936116224 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036180541624874623, + "loss": 2.4672, + "theoretical_loss": 3.6723040708932806, + "tokens_seen": 936181760 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036179538615847546, + "loss": 2.6237, + "theoretical_loss": 3.672279307954521, + "tokens_seen": 936247296 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036178535606820464, + "loss": 2.7813, + "theoretical_loss": 3.6722545472343775, + "tokens_seen": 936312832 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617753259779338, + "loss": 2.5787, + "theoretical_loss": 3.672229788732495, + "tokens_seen": 936378368 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361765295887663, + "loss": 2.676, + "theoretical_loss": 3.67220503244852, + "tokens_seen": 936443904 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617552657973922, + "loss": 2.5799, + "theoretical_loss": 3.6721802783820987, + "tokens_seen": 936509440 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036174523570712137, + "loss": 2.7577, + "theoretical_loss": 3.6721555265328774, + "tokens_seen": 936574976 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617352056168506, + "loss": 2.7343, + "theoretical_loss": 3.6721307769005023, + "tokens_seen": 936640512 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036172517552657973, + "loss": 2.6231, + "theoretical_loss": 3.6721060294846195, + "tokens_seen": 936706048 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036171514543630897, + "loss": 2.4971, + "theoretical_loss": 3.6720812842848756, + "tokens_seen": 936771584 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003617051153460381, + "loss": 2.7271, + "theoretical_loss": 3.6720565413009174, + "tokens_seen": 936837120 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036169508525576733, + "loss": 2.8992, + "theoretical_loss": 3.672031800532391, + "tokens_seen": 936902656 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616850551654965, + "loss": 2.5012, + "theoretical_loss": 3.6720070619789436, + "tokens_seen": 936968192 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616750250752257, + "loss": 2.7448, + "theoretical_loss": 3.671982325640222, + "tokens_seen": 937033728 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036166499498495487, + "loss": 2.5139, + "theoretical_loss": 3.6719575915158726, + "tokens_seen": 937099264 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 524349, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2419161796569824, + "objective/train/theoretical_loss": 3.6719328596055423, + "objective/train/tokens_used": 957624800, + "theoretical_loss": 3.6719328596055423, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036165496489468405, + "loss": 2.5676, + "theoretical_loss": 3.6719328596055423, + "tokens_seen": 937164800 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036164493480441323, + "loss": 2.8033, + "theoretical_loss": 3.6719081299088785, + "tokens_seen": 937230336 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036163490471414247, + "loss": 2.5401, + "theoretical_loss": 3.671883402425528, + "tokens_seen": 937295872 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003616248746238716, + "loss": 2.4762, + "theoretical_loss": 3.671858677155138, + "tokens_seen": 937361408 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036161484453360083, + "loss": 2.4731, + "theoretical_loss": 3.671833954097356, + "tokens_seen": 937426944 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036160481444333, + "loss": 2.777, + "theoretical_loss": 3.6718092332518295, + "tokens_seen": 937492480 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036160481444333, + "loss": 2.7376, + "theoretical_loss": 3.671784514618205, + "tokens_seen": 937558016 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615947843530592, + "loss": 2.9631, + "theoretical_loss": 3.671759798196131, + "tokens_seen": 937623552 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615847542627884, + "loss": 2.8124, + "theoretical_loss": 3.6717350839852547, + "tokens_seen": 937689088 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036157472417251756, + "loss": 2.8745, + "theoretical_loss": 3.671710371985223, + "tokens_seen": 937754624 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036156469408224674, + "loss": 2.65, + "theoretical_loss": 3.6716856621956855, + "tokens_seen": 937820160 + }, + { + "epoch": 0.28, + "learning_rate": 0.000361554663991976, + "loss": 2.861, + "theoretical_loss": 3.6716609546162884, + "tokens_seen": 937885696 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615446339017051, + "loss": 2.6636, + "theoretical_loss": 3.6716362492466796, + "tokens_seen": 937951232 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036153460381143434, + "loss": 2.8794, + "theoretical_loss": 3.6716115460865084, + "tokens_seen": 938016768 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036152457372116346, + "loss": 2.7265, + "theoretical_loss": 3.6715868451354217, + "tokens_seen": 938082304 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615145436308927, + "loss": 2.5481, + "theoretical_loss": 3.671562146393068, + "tokens_seen": 938147840 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003615045135406219, + "loss": 2.7664, + "theoretical_loss": 3.671537449859095, + "tokens_seen": 938213376 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036149448345035106, + "loss": 2.7306, + "theoretical_loss": 3.6715127555331524, + "tokens_seen": 938278912 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036148445336008024, + "loss": 2.5884, + "theoretical_loss": 3.671488063414887, + "tokens_seen": 938344448 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614744232698094, + "loss": 2.6665, + "theoretical_loss": 3.671463373503948, + "tokens_seen": 938409984 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614643931795386, + "loss": 2.906, + "theoretical_loss": 3.671438685799984, + "tokens_seen": 938475520 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036145436308926784, + "loss": 2.8406, + "theoretical_loss": 3.671414000302644, + "tokens_seen": 938541056 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036144433299899697, + "loss": 2.7746, + "theoretical_loss": 3.6713893170115757, + "tokens_seen": 938606592 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614343029087262, + "loss": 2.8227, + "theoretical_loss": 3.6713646359264285, + "tokens_seen": 938672128 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003614242728184554, + "loss": 2.6755, + "theoretical_loss": 3.6713399570468512, + "tokens_seen": 938737664 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 525468, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.555947780609131, + "objective/train/theoretical_loss": 3.671315280372493, + "objective/train/tokens_used": 959263200, + "theoretical_loss": 3.671315280372493, + "tokens_seen": 938803200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036141424272818456, + "loss": 2.602, + "theoretical_loss": 3.671315280372493, + "tokens_seen": 938803200 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036140421263791374, + "loss": 2.8354, + "theoretical_loss": 3.6712906059030024, + "tokens_seen": 938868736 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613941825476429, + "loss": 2.774, + "theoretical_loss": 3.6712659336380287, + "tokens_seen": 938934272 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613841524573721, + "loss": 2.714, + "theoretical_loss": 3.671241263577221, + "tokens_seen": 938999808 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036137412236710134, + "loss": 2.7993, + "theoretical_loss": 3.671216595720229, + "tokens_seen": 939065344 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036136409227683047, + "loss": 2.8738, + "theoretical_loss": 3.671191930066702, + "tokens_seen": 939130880 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613540621865597, + "loss": 2.5938, + "theoretical_loss": 3.6711672666162887, + "tokens_seen": 939196416 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036134403209628883, + "loss": 2.7551, + "theoretical_loss": 3.6711426053686393, + "tokens_seen": 939261952 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036133400200601807, + "loss": 2.9425, + "theoretical_loss": 3.6711179463234034, + "tokens_seen": 939327488 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036132397191574725, + "loss": 2.5775, + "theoretical_loss": 3.6710932894802304, + "tokens_seen": 939393024 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036131394182547643, + "loss": 2.737, + "theoretical_loss": 3.67106863483877, + "tokens_seen": 939458560 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003613039117352056, + "loss": 2.7386, + "theoretical_loss": 3.6710439823986722, + "tokens_seen": 939524096 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036129388164493484, + "loss": 2.7826, + "theoretical_loss": 3.6710193321595868, + "tokens_seen": 939589632 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036128385155466397, + "loss": 2.5974, + "theoretical_loss": 3.670994684121164, + "tokens_seen": 939655168 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612738214643932, + "loss": 2.8526, + "theoretical_loss": 3.6709700382830537, + "tokens_seen": 939720704 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036126379137412233, + "loss": 2.3337, + "theoretical_loss": 3.6709453946449058, + "tokens_seen": 939786240 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036125376128385157, + "loss": 2.8718, + "theoretical_loss": 3.6709207532063712, + "tokens_seen": 939851776 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036124373119358075, + "loss": 2.7124, + "theoretical_loss": 3.6708961139670997, + "tokens_seen": 939917312 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036123370110330993, + "loss": 2.8604, + "theoretical_loss": 3.6708714769267417, + "tokens_seen": 939982848 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612236710130391, + "loss": 2.9014, + "theoretical_loss": 3.6708468420849476, + "tokens_seen": 940048384 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612136409227683, + "loss": 2.5795, + "theoretical_loss": 3.6708222094413685, + "tokens_seen": 940113920 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003612036108324975, + "loss": 2.7011, + "theoretical_loss": 3.670797578995655, + "tokens_seen": 940179456 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611935807422267, + "loss": 2.8098, + "theoretical_loss": 3.6707729507474567, + "tokens_seen": 940244992 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036118355065195584, + "loss": 2.5256, + "theoretical_loss": 3.670748324696426, + "tokens_seen": 940310528 + }, + { + "epoch": 0.28, + "learning_rate": 0.00036117352056168507, + "loss": 2.6763, + "theoretical_loss": 3.6707237008422116, + "tokens_seen": 940376064 + }, + { + "epoch": 0.28, + "objective/train/docs_used": 526013, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.817310333251953, + "objective/train/theoretical_loss": 3.670699079184467, + "objective/train/tokens_used": 960901600, + "theoretical_loss": 3.670699079184467, + "tokens_seen": 940441600 + }, + { + "epoch": 0.28, + "learning_rate": 0.0003611634904714142, + "loss": 2.6014, + "theoretical_loss": 3.670699079184467, + "tokens_seen": 940441600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036115346038114343, + "loss": 2.648, + "theoretical_loss": 3.6706744597228416, + "tokens_seen": 940507136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611434302908726, + "loss": 2.6984, + "theoretical_loss": 3.670649842456987, + "tokens_seen": 940572672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611334002006018, + "loss": 2.5761, + "theoretical_loss": 3.6706252273865543, + "tokens_seen": 940638208 + }, + { + "epoch": 0.29, + "learning_rate": 0.000361123370110331, + "loss": 2.6609, + "theoretical_loss": 3.6706006145111942, + "tokens_seen": 940703744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611133400200602, + "loss": 2.7398, + "theoretical_loss": 3.6705760038305595, + "tokens_seen": 940769280 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003611033099297894, + "loss": 2.6214, + "theoretical_loss": 3.6705513953443, + "tokens_seen": 940834816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610932798395186, + "loss": 2.7006, + "theoretical_loss": 3.6705267890520688, + "tokens_seen": 940900352 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036108324974924776, + "loss": 2.687, + "theoretical_loss": 3.670502184953516, + "tokens_seen": 940965888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036107321965897694, + "loss": 2.4898, + "theoretical_loss": 3.670477583048294, + "tokens_seen": 941031424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610631895687062, + "loss": 2.7117, + "theoretical_loss": 3.670452983336055, + "tokens_seen": 941096960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610531594784353, + "loss": 2.6429, + "theoretical_loss": 3.6704283858164497, + "tokens_seen": 941162496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036104312938816454, + "loss": 2.6568, + "theoretical_loss": 3.6704037904891305, + "tokens_seen": 941228032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036103309929789366, + "loss": 2.7248, + "theoretical_loss": 3.67037919735375, + "tokens_seen": 941293568 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610230692076229, + "loss": 2.8445, + "theoretical_loss": 3.6703546064099593, + "tokens_seen": 941359104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003610130391173521, + "loss": 2.8391, + "theoretical_loss": 3.6703300176574114, + "tokens_seen": 941424640 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036100300902708126, + "loss": 2.8193, + "theoretical_loss": 3.6703054310957577, + "tokens_seen": 941490176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036099297893681044, + "loss": 2.5903, + "theoretical_loss": 3.6702808467246513, + "tokens_seen": 941555712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609829488465396, + "loss": 2.6982, + "theoretical_loss": 3.670256264543744, + "tokens_seen": 941621248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609729187562688, + "loss": 2.7354, + "theoretical_loss": 3.6702316845526886, + "tokens_seen": 941686784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036096288866599804, + "loss": 2.869, + "theoretical_loss": 3.6702071067511373, + "tokens_seen": 941752320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036095285857572717, + "loss": 2.8926, + "theoretical_loss": 3.670182531138743, + "tokens_seen": 941817856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609428284854564, + "loss": 2.8551, + "theoretical_loss": 3.6701579577151584, + "tokens_seen": 941883392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609327983951856, + "loss": 2.7154, + "theoretical_loss": 3.670133386480036, + "tokens_seen": 941948928 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036092276830491476, + "loss": 2.5979, + "theoretical_loss": 3.6701088174330287, + "tokens_seen": 942014464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 527234, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.644130229949951, + "objective/train/theoretical_loss": 3.6700842505737903, + "objective/train/tokens_used": 962540000, + "theoretical_loss": 3.6700842505737903, + "tokens_seen": 942080000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036091273821464394, + "loss": 2.6034, + "theoretical_loss": 3.6700842505737903, + "tokens_seen": 942080000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003609027081243731, + "loss": 2.6743, + "theoretical_loss": 3.6700596859019723, + "tokens_seen": 942145536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608926780341023, + "loss": 2.5416, + "theoretical_loss": 3.670035123417229, + "tokens_seen": 942211072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036088264794383154, + "loss": 2.5893, + "theoretical_loss": 3.670010563119213, + "tokens_seen": 942276608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036087261785356067, + "loss": 2.5442, + "theoretical_loss": 3.669986005007578, + "tokens_seen": 942342144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608625877632899, + "loss": 2.6865, + "theoretical_loss": 3.669961449081977, + "tokens_seen": 942407680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036085255767301903, + "loss": 2.372, + "theoretical_loss": 3.6699368953420635, + "tokens_seen": 942473216 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036084252758274827, + "loss": 2.467, + "theoretical_loss": 3.669912343787491, + "tokens_seen": 942538752 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036083249749247745, + "loss": 2.874, + "theoretical_loss": 3.669887794417913, + "tokens_seen": 942604288 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036082246740220663, + "loss": 2.83, + "theoretical_loss": 3.669863247232983, + "tokens_seen": 942669824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003608124373119358, + "loss": 2.7182, + "theoretical_loss": 3.669838702232355, + "tokens_seen": 942735360 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036080240722166505, + "loss": 2.5017, + "theoretical_loss": 3.669814159415683, + "tokens_seen": 942800896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036079237713139417, + "loss": 2.6341, + "theoretical_loss": 3.66978961878262, + "tokens_seen": 942866432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607823470411234, + "loss": 2.7817, + "theoretical_loss": 3.6697650803328212, + "tokens_seen": 942931968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036077231695085253, + "loss": 2.6504, + "theoretical_loss": 3.66974054406594, + "tokens_seen": 942997504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036076228686058177, + "loss": 2.7501, + "theoretical_loss": 3.66971600998163, + "tokens_seen": 943063040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036075225677031095, + "loss": 2.8004, + "theoretical_loss": 3.669691478079546, + "tokens_seen": 943128576 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036074222668004013, + "loss": 2.7057, + "theoretical_loss": 3.6696669483593425, + "tokens_seen": 943194112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607321965897693, + "loss": 2.4509, + "theoretical_loss": 3.669642420820674, + "tokens_seen": 943259648 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607221664994985, + "loss": 2.7561, + "theoretical_loss": 3.669617895463194, + "tokens_seen": 943325184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607121364092277, + "loss": 2.3886, + "theoretical_loss": 3.6695933722865575, + "tokens_seen": 943390720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003607021063189569, + "loss": 2.7888, + "theoretical_loss": 3.6695688512904194, + "tokens_seen": 943456256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036069207622868604, + "loss": 2.7435, + "theoretical_loss": 3.669544332474434, + "tokens_seen": 943521792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036068204613841527, + "loss": 2.5464, + "theoretical_loss": 3.669519815838256, + "tokens_seen": 943587328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606720160481444, + "loss": 2.7557, + "theoretical_loss": 3.6694953013815406, + "tokens_seen": 943652864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 528141, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.465517044067383, + "objective/train/theoretical_loss": 3.669470789103942, + "objective/train/tokens_used": 964178400, + "theoretical_loss": 3.669470789103942, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036066198595787364, + "loss": 2.7745, + "theoretical_loss": 3.669470789103942, + "tokens_seen": 943718400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606519558676028, + "loss": 2.8102, + "theoretical_loss": 3.669446279005116, + "tokens_seen": 943783936 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360641925777332, + "loss": 2.6492, + "theoretical_loss": 3.6694217710847177, + "tokens_seen": 943849472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606318956870612, + "loss": 2.8314, + "theoretical_loss": 3.6693972653424014, + "tokens_seen": 943915008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606218655967904, + "loss": 2.6344, + "theoretical_loss": 3.669372761777823, + "tokens_seen": 943980544 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036061183550651954, + "loss": 2.7509, + "theoretical_loss": 3.669348260390638, + "tokens_seen": 944046080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003606018054162488, + "loss": 3.0374, + "theoretical_loss": 3.669323761180501, + "tokens_seen": 944111616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605917753259779, + "loss": 2.965, + "theoretical_loss": 3.6692992641470683, + "tokens_seen": 944177152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036058174523570714, + "loss": 2.8973, + "theoretical_loss": 3.6692747692899945, + "tokens_seen": 944242688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605717151454363, + "loss": 2.8169, + "theoretical_loss": 3.6692502766089357, + "tokens_seen": 944308224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605616850551655, + "loss": 2.7218, + "theoretical_loss": 3.669225786103548, + "tokens_seen": 944373760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605516549648947, + "loss": 2.5733, + "theoretical_loss": 3.6692012977734865, + "tokens_seen": 944439296 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036054162487462386, + "loss": 2.6159, + "theoretical_loss": 3.6691768116184074, + "tokens_seen": 944504832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036053159478435304, + "loss": 2.5814, + "theoretical_loss": 3.669152327637967, + "tokens_seen": 944570368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605215646940823, + "loss": 2.8545, + "theoretical_loss": 3.66912784583182, + "tokens_seen": 944635904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003605115346038114, + "loss": 2.841, + "theoretical_loss": 3.669103366199624, + "tokens_seen": 944701440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036050150451354064, + "loss": 2.5827, + "theoretical_loss": 3.6690788887410344, + "tokens_seen": 944766976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036049147442326977, + "loss": 2.6336, + "theoretical_loss": 3.6690544134557075, + "tokens_seen": 944832512 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360481444332999, + "loss": 2.6132, + "theoretical_loss": 3.6690299403432998, + "tokens_seen": 944898048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604714142427282, + "loss": 2.5371, + "theoretical_loss": 3.6690054694034675, + "tokens_seen": 944963584 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036046138415245737, + "loss": 2.5715, + "theoretical_loss": 3.6689810006358674, + "tokens_seen": 945029120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036045135406218655, + "loss": 2.7622, + "theoretical_loss": 3.6689565340401558, + "tokens_seen": 945094656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604413239719158, + "loss": 2.835, + "theoretical_loss": 3.6689320696159893, + "tokens_seen": 945160192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604312938816449, + "loss": 2.6585, + "theoretical_loss": 3.6689076073630242, + "tokens_seen": 945225728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036042126379137414, + "loss": 2.9013, + "theoretical_loss": 3.6688831472809182, + "tokens_seen": 945291264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 528757, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7434470653533936, + "objective/train/theoretical_loss": 3.668858689369327, + "objective/train/tokens_used": 965816800, + "theoretical_loss": 3.668858689369327, + "tokens_seen": 945356800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036041123370110327, + "loss": 2.6086, + "theoretical_loss": 3.668858689369327, + "tokens_seen": 945356800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003604012036108325, + "loss": 2.698, + "theoretical_loss": 3.6688342336279094, + "tokens_seen": 945422336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003603911735205617, + "loss": 2.7111, + "theoretical_loss": 3.6688097800563213, + "tokens_seen": 945487872 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036038114343029087, + "loss": 2.7297, + "theoretical_loss": 3.668785328654219, + "tokens_seen": 945553408 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036037111334002005, + "loss": 2.8327, + "theoretical_loss": 3.668760879421261, + "tokens_seen": 945618944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036036108324974923, + "loss": 2.6444, + "theoretical_loss": 3.668736432357104, + "tokens_seen": 945684480 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036035105315947847, + "loss": 2.7526, + "theoretical_loss": 3.6687119874614047, + "tokens_seen": 945750016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036034102306920765, + "loss": 2.7224, + "theoretical_loss": 3.668687544733822, + "tokens_seen": 945815552 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036033099297893683, + "loss": 2.6261, + "theoretical_loss": 3.668663104174012, + "tokens_seen": 945881088 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360320962888666, + "loss": 2.662, + "theoretical_loss": 3.668638665781633, + "tokens_seen": 945946624 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036031093279839525, + "loss": 2.6121, + "theoretical_loss": 3.6686142295563426, + "tokens_seen": 946012160 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036030090270812437, + "loss": 2.791, + "theoretical_loss": 3.668589795497799, + "tokens_seen": 946077696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602908726178536, + "loss": 2.6844, + "theoretical_loss": 3.6685653636056585, + "tokens_seen": 946143232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036028084252758273, + "loss": 2.9634, + "theoretical_loss": 3.66854093387958, + "tokens_seen": 946208768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036027081243731197, + "loss": 2.7207, + "theoretical_loss": 3.668516506319222, + "tokens_seen": 946274304 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036026078234704115, + "loss": 2.5772, + "theoretical_loss": 3.668492080924241, + "tokens_seen": 946339840 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036025075225677033, + "loss": 2.6456, + "theoretical_loss": 3.668467657694297, + "tokens_seen": 946405376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602407221664995, + "loss": 2.7981, + "theoretical_loss": 3.668443236629047, + "tokens_seen": 946470912 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602306920762287, + "loss": 2.8633, + "theoretical_loss": 3.668418817728149, + "tokens_seen": 946536448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602206619859579, + "loss": 2.505, + "theoretical_loss": 3.6683944009912617, + "tokens_seen": 946601984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003602106318956871, + "loss": 2.7682, + "theoretical_loss": 3.6683699864180443, + "tokens_seen": 946667520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036020060180541624, + "loss": 2.6689, + "theoretical_loss": 3.6683455740081543, + "tokens_seen": 946733056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601905717151455, + "loss": 2.7219, + "theoretical_loss": 3.6683211637612505, + "tokens_seen": 946798592 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601805416248746, + "loss": 2.8951, + "theoretical_loss": 3.668296755676992, + "tokens_seen": 946864128 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036017051153460384, + "loss": 2.8538, + "theoretical_loss": 3.668272349755037, + "tokens_seen": 946929664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 529862, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7047557830810547, + "objective/train/theoretical_loss": 3.6682479459950446, + "objective/train/tokens_used": 967455200, + "theoretical_loss": 3.6682479459950446, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360160481444333, + "loss": 2.571, + "theoretical_loss": 3.6682479459950446, + "tokens_seen": 946995200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601504513540622, + "loss": 2.64, + "theoretical_loss": 3.6682235443966733, + "tokens_seen": 947060736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601404212637914, + "loss": 2.7239, + "theoretical_loss": 3.6681991449595825, + "tokens_seen": 947126272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601303911735206, + "loss": 2.706, + "theoretical_loss": 3.668174747683431, + "tokens_seen": 947191808 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036012036108324974, + "loss": 2.7165, + "theoretical_loss": 3.6681503525678787, + "tokens_seen": 947257344 + }, + { + "epoch": 0.29, + "learning_rate": 0.000360110330992979, + "loss": 2.756, + "theoretical_loss": 3.668125959612584, + "tokens_seen": 947322880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003601003009027081, + "loss": 2.5381, + "theoretical_loss": 3.668101568817206, + "tokens_seen": 947388416 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036009027081243734, + "loss": 2.6841, + "theoretical_loss": 3.6680771801814043, + "tokens_seen": 947453952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600802407221665, + "loss": 2.7336, + "theoretical_loss": 3.668052793704838, + "tokens_seen": 947519488 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600702106318957, + "loss": 2.679, + "theoretical_loss": 3.668028409387168, + "tokens_seen": 947585024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600601805416249, + "loss": 2.7866, + "theoretical_loss": 3.668004027228052, + "tokens_seen": 947650560 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036005015045135406, + "loss": 2.6878, + "theoretical_loss": 3.667979647227151, + "tokens_seen": 947716096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036004012036108324, + "loss": 2.6239, + "theoretical_loss": 3.6679552693841244, + "tokens_seen": 947781632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600300902708125, + "loss": 2.6908, + "theoretical_loss": 3.6679308936986317, + "tokens_seen": 947847168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003600200601805416, + "loss": 2.5497, + "theoretical_loss": 3.667906520170333, + "tokens_seen": 947912704 + }, + { + "epoch": 0.29, + "learning_rate": 0.00036001003009027084, + "loss": 2.7513, + "theoretical_loss": 3.6678821487988884, + "tokens_seen": 947978240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035999999999999997, + "loss": 2.3677, + "theoretical_loss": 3.667857779583958, + "tokens_seen": 948043776 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599899699097292, + "loss": 2.6004, + "theoretical_loss": 3.6678334125252015, + "tokens_seen": 948109312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599799398194584, + "loss": 2.6877, + "theoretical_loss": 3.667809047622279, + "tokens_seen": 948174848 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035996990972918757, + "loss": 2.8092, + "theoretical_loss": 3.667784684874851, + "tokens_seen": 948240384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035995987963891675, + "loss": 2.6486, + "theoretical_loss": 3.667760324282579, + "tokens_seen": 948305920 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359949849548646, + "loss": 2.5051, + "theoretical_loss": 3.6677359658451216, + "tokens_seen": 948371456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599398194583751, + "loss": 2.7112, + "theoretical_loss": 3.6677116095621405, + "tokens_seen": 948436992 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035992978936810434, + "loss": 2.5369, + "theoretical_loss": 3.6676872554332958, + "tokens_seen": 948502528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035991975927783347, + "loss": 2.6575, + "theoretical_loss": 3.667662903458248, + "tokens_seen": 948568064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 530380, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.964254140853882, + "objective/train/theoretical_loss": 3.6676385536366585, + "objective/train/tokens_used": 969093600, + "theoretical_loss": 3.6676385536366585, + "tokens_seen": 948633600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003599097291875627, + "loss": 2.7343, + "theoretical_loss": 3.6676385536366585, + "tokens_seen": 948633600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598996990972919, + "loss": 2.7499, + "theoretical_loss": 3.6676142059681878, + "tokens_seen": 948699136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035988966900702107, + "loss": 2.8896, + "theoretical_loss": 3.6675898604524964, + "tokens_seen": 948764672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035987963891675025, + "loss": 2.9244, + "theoretical_loss": 3.667565517089246, + "tokens_seen": 948830208 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035986960882647943, + "loss": 2.5677, + "theoretical_loss": 3.667541175878097, + "tokens_seen": 948895744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598595787362086, + "loss": 2.8888, + "theoretical_loss": 3.6675168368187103, + "tokens_seen": 948961280 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035984954864593785, + "loss": 2.5331, + "theoretical_loss": 3.6674924999107485, + "tokens_seen": 949026816 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359839518555667, + "loss": 2.8319, + "theoretical_loss": 3.6674681651538714, + "tokens_seen": 949092352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003598294884653962, + "loss": 2.6815, + "theoretical_loss": 3.6674438325477405, + "tokens_seen": 949157888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035981945837512534, + "loss": 2.7486, + "theoretical_loss": 3.6674195020920184, + "tokens_seen": 949223424 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035980942828485457, + "loss": 2.7056, + "theoretical_loss": 3.6673951737863657, + "tokens_seen": 949288960 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035979939819458375, + "loss": 2.6764, + "theoretical_loss": 3.667370847630444, + "tokens_seen": 949354496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035978936810431293, + "loss": 2.5544, + "theoretical_loss": 3.667346523623915, + "tokens_seen": 949420032 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597793380140421, + "loss": 2.6938, + "theoretical_loss": 3.6673222017664404, + "tokens_seen": 949485568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035976930792377135, + "loss": 2.4797, + "theoretical_loss": 3.6672978820576825, + "tokens_seen": 949551104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597592778335005, + "loss": 2.8528, + "theoretical_loss": 3.667273564497303, + "tokens_seen": 949616640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597492477432297, + "loss": 2.6105, + "theoretical_loss": 3.6672492490849633, + "tokens_seen": 949682176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035973921765295884, + "loss": 2.5964, + "theoretical_loss": 3.667224935820326, + "tokens_seen": 949747712 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003597291875626881, + "loss": 2.7149, + "theoretical_loss": 3.667200624703053, + "tokens_seen": 949813248 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035971915747241726, + "loss": 2.5913, + "theoretical_loss": 3.6671763157328066, + "tokens_seen": 949878784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035970912738214644, + "loss": 2.6368, + "theoretical_loss": 3.667152008909249, + "tokens_seen": 949944320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596990972918756, + "loss": 2.7971, + "theoretical_loss": 3.6671277042320423, + "tokens_seen": 950009856 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596890672016048, + "loss": 2.7024, + "theoretical_loss": 3.6671034017008495, + "tokens_seen": 950075392 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359679037111334, + "loss": 2.5597, + "theoretical_loss": 3.667079101315333, + "tokens_seen": 950140928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596690070210632, + "loss": 2.7964, + "theoretical_loss": 3.667054803075155, + "tokens_seen": 950206464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 531473, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.32260799407959, + "objective/train/theoretical_loss": 3.6670305069799785, + "objective/train/tokens_used": 970732000, + "theoretical_loss": 3.6670305069799785, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035965897693079234, + "loss": 2.6143, + "theoretical_loss": 3.6670305069799785, + "tokens_seen": 950272000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596489468405216, + "loss": 2.657, + "theoretical_loss": 3.667006213029466, + "tokens_seen": 950337536 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035963891675025076, + "loss": 2.7829, + "theoretical_loss": 3.6669819212232806, + "tokens_seen": 950403072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035962888665997994, + "loss": 2.7387, + "theoretical_loss": 3.6669576315610843, + "tokens_seen": 950468608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596188565697091, + "loss": 2.7885, + "theoretical_loss": 3.6669333440425413, + "tokens_seen": 950534144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003596088264794383, + "loss": 2.854, + "theoretical_loss": 3.666909058667314, + "tokens_seen": 950599680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035959879638916754, + "loss": 2.5672, + "theoretical_loss": 3.666884775435066, + "tokens_seen": 950665216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595887662988967, + "loss": 2.4754, + "theoretical_loss": 3.6668604943454595, + "tokens_seen": 950730752 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595787362086259, + "loss": 2.856, + "theoretical_loss": 3.6668362153981584, + "tokens_seen": 950796288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595687061183551, + "loss": 2.6182, + "theoretical_loss": 3.6668119385928257, + "tokens_seen": 950861824 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035955867602808426, + "loss": 2.7393, + "theoretical_loss": 3.6667876639291257, + "tokens_seen": 950927360 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035954864593781344, + "loss": 2.5813, + "theoretical_loss": 3.6667633914067213, + "tokens_seen": 950992896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595386158475427, + "loss": 2.7133, + "theoretical_loss": 3.6667391210252758, + "tokens_seen": 951058432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003595285857572718, + "loss": 2.5476, + "theoretical_loss": 3.666714852784453, + "tokens_seen": 951123968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035951855566700104, + "loss": 2.8157, + "theoretical_loss": 3.6666905866839175, + "tokens_seen": 951189504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035950852557673017, + "loss": 2.8043, + "theoretical_loss": 3.6666663227233314, + "tokens_seen": 951255040 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594984954864594, + "loss": 2.7221, + "theoretical_loss": 3.66664206090236, + "tokens_seen": 951320576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594884653961886, + "loss": 2.758, + "theoretical_loss": 3.6666178012206663, + "tokens_seen": 951386112 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035947843530591777, + "loss": 2.7675, + "theoretical_loss": 3.666593543677915, + "tokens_seen": 951451648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035946840521564695, + "loss": 2.677, + "theoretical_loss": 3.66656928827377, + "tokens_seen": 951517184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594583751253762, + "loss": 2.8365, + "theoretical_loss": 3.6665450350078954, + "tokens_seen": 951582720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594483450351053, + "loss": 2.7003, + "theoretical_loss": 3.666520783879956, + "tokens_seen": 951648256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035943831494483454, + "loss": 2.7854, + "theoretical_loss": 3.6664965348896157, + "tokens_seen": 951713792 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035942828485456367, + "loss": 2.5429, + "theoretical_loss": 3.6664722880365384, + "tokens_seen": 951779328 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594182547642929, + "loss": 2.8141, + "theoretical_loss": 3.666448043320389, + "tokens_seen": 951844864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 532856, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.025040864944458, + "objective/train/theoretical_loss": 3.666423800740832, + "objective/train/tokens_used": 972370400, + "theoretical_loss": 3.666423800740832, + "tokens_seen": 951910400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003594082246740221, + "loss": 2.7602, + "theoretical_loss": 3.666423800740832, + "tokens_seen": 951910400 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035939819458375127, + "loss": 2.6914, + "theoretical_loss": 3.666399560297532, + "tokens_seen": 951975936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035938816449348045, + "loss": 2.6439, + "theoretical_loss": 3.6663753219901545, + "tokens_seen": 952041472 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035937813440320963, + "loss": 2.8814, + "theoretical_loss": 3.6663510858183628, + "tokens_seen": 952107008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593681043129388, + "loss": 2.5812, + "theoretical_loss": 3.6663268517818235, + "tokens_seen": 952172544 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035935807422266805, + "loss": 2.9106, + "theoretical_loss": 3.6663026198802, + "tokens_seen": 952238080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593480441323972, + "loss": 2.9617, + "theoretical_loss": 3.666278390113158, + "tokens_seen": 952303616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003593380140421264, + "loss": 2.798, + "theoretical_loss": 3.6662541624803624, + "tokens_seen": 952369152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035932798395185554, + "loss": 2.7121, + "theoretical_loss": 3.6662299369814786, + "tokens_seen": 952434688 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035931795386158477, + "loss": 2.6545, + "theoretical_loss": 3.6662057136161716, + "tokens_seen": 952500224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035930792377131395, + "loss": 2.729, + "theoretical_loss": 3.6661814923841067, + "tokens_seen": 952565760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035929789368104313, + "loss": 2.8024, + "theoretical_loss": 3.6661572732849494, + "tokens_seen": 952631296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592878635907723, + "loss": 2.5495, + "theoretical_loss": 3.6661330563183654, + "tokens_seen": 952696832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035927783350050155, + "loss": 2.4999, + "theoretical_loss": 3.66610884148402, + "tokens_seen": 952762368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592678034102307, + "loss": 2.8948, + "theoretical_loss": 3.6660846287815794, + "tokens_seen": 952827904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592577733199599, + "loss": 2.654, + "theoretical_loss": 3.6660604182107077, + "tokens_seen": 952893440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035924774322968904, + "loss": 2.8302, + "theoretical_loss": 3.666036209771072, + "tokens_seen": 952958976 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592377131394183, + "loss": 2.8485, + "theoretical_loss": 3.6660120034623382, + "tokens_seen": 953024512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035922768304914746, + "loss": 2.7843, + "theoretical_loss": 3.665987799284171, + "tokens_seen": 953090048 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035921765295887664, + "loss": 2.351, + "theoretical_loss": 3.665963597236238, + "tokens_seen": 953155584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003592076228686058, + "loss": 2.6998, + "theoretical_loss": 3.6659393973182044, + "tokens_seen": 953221120 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359197592778335, + "loss": 2.5819, + "theoretical_loss": 3.665915199529736, + "tokens_seen": 953286656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591875626880642, + "loss": 2.6821, + "theoretical_loss": 3.6658910038704997, + "tokens_seen": 953352192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591775325977934, + "loss": 2.7457, + "theoretical_loss": 3.665866810340162, + "tokens_seen": 953417728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035916750250752254, + "loss": 2.6965, + "theoretical_loss": 3.665842618938388, + "tokens_seen": 953483264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 533452, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.747199773788452, + "objective/train/theoretical_loss": 3.6658184296648457, + "objective/train/tokens_used": 974008800, + "theoretical_loss": 3.6658184296648457, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591574724172518, + "loss": 2.8721, + "theoretical_loss": 3.6658184296648457, + "tokens_seen": 953548800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035914744232698096, + "loss": 2.6881, + "theoretical_loss": 3.6657942425192003, + "tokens_seen": 953614336 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035913741223671014, + "loss": 2.6932, + "theoretical_loss": 3.665770057501119, + "tokens_seen": 953679872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591273821464393, + "loss": 2.7589, + "theoretical_loss": 3.665745874610269, + "tokens_seen": 953745408 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591173520561685, + "loss": 2.8227, + "theoretical_loss": 3.665721693846316, + "tokens_seen": 953810944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003591073219658977, + "loss": 2.6922, + "theoretical_loss": 3.6656975152089277, + "tokens_seen": 953876480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590972918756269, + "loss": 2.6638, + "theoretical_loss": 3.6656733386977702, + "tokens_seen": 953942016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035908726178535605, + "loss": 2.9033, + "theoretical_loss": 3.665649164312511, + "tokens_seen": 954007552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590772316950853, + "loss": 2.6242, + "theoretical_loss": 3.665624992052817, + "tokens_seen": 954073088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590672016048144, + "loss": 2.7591, + "theoretical_loss": 3.665600821918355, + "tokens_seen": 954138624 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035905717151454364, + "loss": 2.748, + "theoretical_loss": 3.6655766539087935, + "tokens_seen": 954204160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590471414242728, + "loss": 2.5731, + "theoretical_loss": 3.6655524880237977, + "tokens_seen": 954269696 + }, + { + "epoch": 0.29, + "learning_rate": 0.000359037111334002, + "loss": 2.6532, + "theoretical_loss": 3.665528324263037, + "tokens_seen": 954335232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003590270812437312, + "loss": 2.5441, + "theoretical_loss": 3.665504162626177, + "tokens_seen": 954400768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035901705115346037, + "loss": 2.5651, + "theoretical_loss": 3.6654800031128865, + "tokens_seen": 954466304 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035900702106318955, + "loss": 2.7077, + "theoretical_loss": 3.665455845722833, + "tokens_seen": 954531840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589969909729188, + "loss": 2.6124, + "theoretical_loss": 3.6654316904556827, + "tokens_seen": 954597376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589869608826479, + "loss": 2.7877, + "theoretical_loss": 3.665407537311105, + "tokens_seen": 954662912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035897693079237715, + "loss": 2.6148, + "theoretical_loss": 3.6653833862887675, + "tokens_seen": 954728448 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035896690070210633, + "loss": 2.7657, + "theoretical_loss": 3.665359237388337, + "tokens_seen": 954793984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589568706118355, + "loss": 2.6572, + "theoretical_loss": 3.6653350906094824, + "tokens_seen": 954859520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589468405215647, + "loss": 2.5009, + "theoretical_loss": 3.6653109459518713, + "tokens_seen": 954925056 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035893681043129387, + "loss": 2.5484, + "theoretical_loss": 3.6652868034151718, + "tokens_seen": 954990592 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035892678034102305, + "loss": 2.7428, + "theoretical_loss": 3.6652626629990523, + "tokens_seen": 955056128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589167502507523, + "loss": 2.4657, + "theoretical_loss": 3.665238524703181, + "tokens_seen": 955121664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 534564, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8826329708099365, + "objective/train/theoretical_loss": 3.6652143885272257, + "objective/train/tokens_used": 975647200, + "theoretical_loss": 3.6652143885272257, + "tokens_seen": 955187200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003589067201604814, + "loss": 2.5385, + "theoretical_loss": 3.6652143885272257, + "tokens_seen": 955187200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035889669007021065, + "loss": 2.8421, + "theoretical_loss": 3.6651902544708554, + "tokens_seen": 955252736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588866599799398, + "loss": 2.7383, + "theoretical_loss": 3.665166122533739, + "tokens_seen": 955318272 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358876629889669, + "loss": 2.716, + "theoretical_loss": 3.6651419927155438, + "tokens_seen": 955383808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588665997993982, + "loss": 2.7816, + "theoretical_loss": 3.665117865015939, + "tokens_seen": 955449344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588565697091274, + "loss": 2.6429, + "theoretical_loss": 3.6650937394345933, + "tokens_seen": 955514880 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003588465396188566, + "loss": 2.7899, + "theoretical_loss": 3.665069615971176, + "tokens_seen": 955580416 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035883650952858574, + "loss": 2.6999, + "theoretical_loss": 3.665045494625355, + "tokens_seen": 955645952 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035882647943831497, + "loss": 2.5648, + "theoretical_loss": 3.6650213753967997, + "tokens_seen": 955711488 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035881644934804415, + "loss": 2.669, + "theoretical_loss": 3.664997258285179, + "tokens_seen": 955777024 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035880641925777333, + "loss": 2.6379, + "theoretical_loss": 3.664973143290162, + "tokens_seen": 955842560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587963891675025, + "loss": 2.6819, + "theoretical_loss": 3.6649490304114183, + "tokens_seen": 955908096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035878635907723175, + "loss": 2.7801, + "theoretical_loss": 3.664924919648616, + "tokens_seen": 955973632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587763289869609, + "loss": 2.6382, + "theoretical_loss": 3.6649008110014254, + "tokens_seen": 956039168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587662988966901, + "loss": 2.8571, + "theoretical_loss": 3.664876704469515, + "tokens_seen": 956104704 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035875626880641924, + "loss": 2.9089, + "theoretical_loss": 3.6648526000525554, + "tokens_seen": 956170240 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587462387161485, + "loss": 2.8262, + "theoretical_loss": 3.6648284977502152, + "tokens_seen": 956235776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035873620862587766, + "loss": 2.8647, + "theoretical_loss": 3.664804397562164, + "tokens_seen": 956301312 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035872617853560684, + "loss": 2.8062, + "theoretical_loss": 3.664780299488072, + "tokens_seen": 956366848 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358716148445336, + "loss": 2.5454, + "theoretical_loss": 3.6647562035276082, + "tokens_seen": 956432384 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003587061183550652, + "loss": 2.7496, + "theoretical_loss": 3.6647321096804433, + "tokens_seen": 956497920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586960882647944, + "loss": 2.6876, + "theoretical_loss": 3.6647080179462463, + "tokens_seen": 956563456 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586860581745236, + "loss": 2.9231, + "theoretical_loss": 3.6646839283246875, + "tokens_seen": 956628992 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035867602808425274, + "loss": 2.6204, + "theoretical_loss": 3.664659840815437, + "tokens_seen": 956694528 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358665997993982, + "loss": 2.719, + "theoretical_loss": 3.664635755418165, + "tokens_seen": 956760064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 535115, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7175190448760986, + "objective/train/theoretical_loss": 3.6646116721325415, + "objective/train/tokens_used": 977285600, + "theoretical_loss": 3.6646116721325415, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035865596790371116, + "loss": 2.5719, + "theoretical_loss": 3.6646116721325415, + "tokens_seen": 956825600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035864593781344034, + "loss": 2.4455, + "theoretical_loss": 3.664587590958237, + "tokens_seen": 956891136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586359077231695, + "loss": 2.6658, + "theoretical_loss": 3.6645635118949214, + "tokens_seen": 956956672 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586258776328987, + "loss": 2.3844, + "theoretical_loss": 3.664539434942265, + "tokens_seen": 957022208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586158475426279, + "loss": 2.6877, + "theoretical_loss": 3.664515360099939, + "tokens_seen": 957087744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003586058174523571, + "loss": 2.7858, + "theoretical_loss": 3.664491287367613, + "tokens_seen": 957153280 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035859578736208625, + "loss": 2.8758, + "theoretical_loss": 3.664467216744959, + "tokens_seen": 957218816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585857572718155, + "loss": 2.7512, + "theoretical_loss": 3.664443148231647, + "tokens_seen": 957284352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585757271815446, + "loss": 2.9092, + "theoretical_loss": 3.6644190818273463, + "tokens_seen": 957349888 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035856569709127384, + "loss": 2.679, + "theoretical_loss": 3.66439501753173, + "tokens_seen": 957415424 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358555667001003, + "loss": 2.9037, + "theoretical_loss": 3.6643709553444683, + "tokens_seen": 957480960 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585456369107322, + "loss": 2.5161, + "theoretical_loss": 3.6643468952652314, + "tokens_seen": 957546496 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003585356068204614, + "loss": 2.5333, + "theoretical_loss": 3.664322837293691, + "tokens_seen": 957612032 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035852557673019057, + "loss": 2.8931, + "theoretical_loss": 3.664298781429518, + "tokens_seen": 957677568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035851554663991975, + "loss": 2.9314, + "theoretical_loss": 3.6642747276723844, + "tokens_seen": 957743104 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358505516549649, + "loss": 2.8517, + "theoretical_loss": 3.66425067602196, + "tokens_seen": 957808640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584954864593781, + "loss": 2.5505, + "theoretical_loss": 3.6642266264779177, + "tokens_seen": 957874176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035848545636910735, + "loss": 2.3913, + "theoretical_loss": 3.6642025790399284, + "tokens_seen": 957939712 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035847542627883653, + "loss": 2.8246, + "theoretical_loss": 3.6641785337076627, + "tokens_seen": 958005248 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584653961885657, + "loss": 2.536, + "theoretical_loss": 3.6641544904807937, + "tokens_seen": 958070784 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584553660982949, + "loss": 2.7839, + "theoretical_loss": 3.664130449358992, + "tokens_seen": 958136320 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035844533600802407, + "loss": 2.7568, + "theoretical_loss": 3.6641064103419296, + "tokens_seen": 958201856 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035843530591775325, + "loss": 2.5894, + "theoretical_loss": 3.6640823734292782, + "tokens_seen": 958267392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584252758274825, + "loss": 2.809, + "theoretical_loss": 3.66405833862071, + "tokens_seen": 958332928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003584152457372116, + "loss": 2.5561, + "theoretical_loss": 3.6640343059158966, + "tokens_seen": 958398464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 536290, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.003551483154297, + "objective/train/theoretical_loss": 3.66401027531451, + "objective/train/tokens_used": 978924000, + "theoretical_loss": 3.66401027531451, + "tokens_seen": 958464000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035840521564694085, + "loss": 2.7357, + "theoretical_loss": 3.66401027531451, + "tokens_seen": 958464000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035839518555667, + "loss": 2.6729, + "theoretical_loss": 3.663986246816223, + "tokens_seen": 958529536 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583851554663992, + "loss": 2.7778, + "theoretical_loss": 3.663962220420707, + "tokens_seen": 958595072 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583751253761284, + "loss": 2.7285, + "theoretical_loss": 3.663938196127634, + "tokens_seen": 958660608 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583650952858576, + "loss": 2.4119, + "theoretical_loss": 3.6639141739366776, + "tokens_seen": 958726144 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035835506519558676, + "loss": 2.7283, + "theoretical_loss": 3.663890153847509, + "tokens_seen": 958791680 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035834503510531594, + "loss": 2.4578, + "theoretical_loss": 3.663866135859801, + "tokens_seen": 958857216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583350050150451, + "loss": 2.7346, + "theoretical_loss": 3.663842119973226, + "tokens_seen": 958922752 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035832497492477435, + "loss": 2.6133, + "theoretical_loss": 3.663818106187457, + "tokens_seen": 958988288 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583149448345035, + "loss": 2.6151, + "theoretical_loss": 3.6637940945021663, + "tokens_seen": 959053824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003583049147442327, + "loss": 2.6634, + "theoretical_loss": 3.663770084917027, + "tokens_seen": 959119360 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582948846539619, + "loss": 2.7547, + "theoretical_loss": 3.663746077431712, + "tokens_seen": 959184896 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582848545636911, + "loss": 2.8404, + "theoretical_loss": 3.6637220720458936, + "tokens_seen": 959250432 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035827482447342026, + "loss": 2.5344, + "theoretical_loss": 3.6636980687592455, + "tokens_seen": 959315968 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035826479438314944, + "loss": 2.7036, + "theoretical_loss": 3.66367406757144, + "tokens_seen": 959381504 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582547642928786, + "loss": 2.6886, + "theoretical_loss": 3.6636500684821507, + "tokens_seen": 959447040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035824473420260786, + "loss": 2.7948, + "theoretical_loss": 3.663626071491051, + "tokens_seen": 959512576 + }, + { + "epoch": 0.29, + "learning_rate": 0.000358234704112337, + "loss": 2.8091, + "theoretical_loss": 3.6636020765978135, + "tokens_seen": 959578112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582246740220662, + "loss": 2.6147, + "theoretical_loss": 3.663578083802112, + "tokens_seen": 959643648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035821464393179535, + "loss": 2.6451, + "theoretical_loss": 3.6635540931036195, + "tokens_seen": 959709184 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003582046138415246, + "loss": 2.6816, + "theoretical_loss": 3.6635301045020103, + "tokens_seen": 959774720 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035819458375125376, + "loss": 2.7626, + "theoretical_loss": 3.663506117996958, + "tokens_seen": 959840256 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035818455366098294, + "loss": 2.5617, + "theoretical_loss": 3.6634821335881345, + "tokens_seen": 959905792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581745235707121, + "loss": 2.6693, + "theoretical_loss": 3.663458151275216, + "tokens_seen": 959971328 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035816449348044136, + "loss": 2.64, + "theoretical_loss": 3.663434171057874, + "tokens_seen": 960036864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 537086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.786989688873291, + "objective/train/theoretical_loss": 3.6634101929357836, + "objective/train/tokens_used": 980562400, + "theoretical_loss": 3.6634101929357836, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581544633901705, + "loss": 2.581, + "theoretical_loss": 3.6634101929357836, + "tokens_seen": 960102400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581444332998997, + "loss": 2.7225, + "theoretical_loss": 3.663386216908619, + "tokens_seen": 960167936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035813440320962885, + "loss": 2.5938, + "theoretical_loss": 3.6633622429760533, + "tokens_seen": 960233472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003581243731193581, + "loss": 2.7526, + "theoretical_loss": 3.6633382711377607, + "tokens_seen": 960299008 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035811434302908727, + "loss": 2.7211, + "theoretical_loss": 3.663314301393416, + "tokens_seen": 960364544 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035810431293881645, + "loss": 2.4448, + "theoretical_loss": 3.663290333742693, + "tokens_seen": 960430080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580942828485457, + "loss": 2.5715, + "theoretical_loss": 3.6632663681852655, + "tokens_seen": 960495616 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580842527582748, + "loss": 2.6514, + "theoretical_loss": 3.663242404720809, + "tokens_seen": 960561152 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035807422266800404, + "loss": 2.6498, + "theoretical_loss": 3.663218443348997, + "tokens_seen": 960626688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580641925777332, + "loss": 2.7838, + "theoretical_loss": 3.663194484069505, + "tokens_seen": 960692224 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580541624874624, + "loss": 2.4934, + "theoretical_loss": 3.6631705268820065, + "tokens_seen": 960757760 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580441323971916, + "loss": 2.6763, + "theoretical_loss": 3.6631465717861764, + "tokens_seen": 960823296 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035803410230692077, + "loss": 2.7096, + "theoretical_loss": 3.6631226187816894, + "tokens_seen": 960888832 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035802407221664995, + "loss": 2.7574, + "theoretical_loss": 3.663098667868221, + "tokens_seen": 960954368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580140421263792, + "loss": 2.3962, + "theoretical_loss": 3.6630747190454453, + "tokens_seen": 961019904 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003580040120361083, + "loss": 2.5524, + "theoretical_loss": 3.6630507723130377, + "tokens_seen": 961085440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035799398194583755, + "loss": 2.7353, + "theoretical_loss": 3.663026827670673, + "tokens_seen": 961150976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035798395185556673, + "loss": 2.6363, + "theoretical_loss": 3.6630028851180265, + "tokens_seen": 961216512 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579739217652959, + "loss": 2.5274, + "theoretical_loss": 3.662978944654773, + "tokens_seen": 961282048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579638916750251, + "loss": 2.7494, + "theoretical_loss": 3.662955006280588, + "tokens_seen": 961347584 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035795386158475427, + "loss": 2.7442, + "theoretical_loss": 3.662931069995147, + "tokens_seen": 961413120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035794383149448345, + "loss": 2.8181, + "theoretical_loss": 3.6629071357981244, + "tokens_seen": 961478656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579338014042127, + "loss": 2.6398, + "theoretical_loss": 3.662883203689197, + "tokens_seen": 961544192 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579237713139418, + "loss": 2.8939, + "theoretical_loss": 3.6628592736680394, + "tokens_seen": 961609728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035791374122367105, + "loss": 2.6428, + "theoretical_loss": 3.6628353457343277, + "tokens_seen": 961675264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 537653, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.703756809234619, + "objective/train/theoretical_loss": 3.6628114198877375, + "objective/train/tokens_used": 982200800, + "theoretical_loss": 3.6628114198877375, + "tokens_seen": 961740800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003579037111334002, + "loss": 2.5893, + "theoretical_loss": 3.6628114198877375, + "tokens_seen": 961740800 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578936810431294, + "loss": 2.5064, + "theoretical_loss": 3.6627874961279447, + "tokens_seen": 961806336 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578836509528586, + "loss": 2.6169, + "theoretical_loss": 3.6627635744546243, + "tokens_seen": 961871872 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578736208625878, + "loss": 2.5647, + "theoretical_loss": 3.662739654867453, + "tokens_seen": 961937408 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035786359077231696, + "loss": 2.7298, + "theoretical_loss": 3.662715737366107, + "tokens_seen": 962002944 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035785356068204614, + "loss": 2.5342, + "theoretical_loss": 3.6626918219502613, + "tokens_seen": 962068480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578435305917753, + "loss": 2.5674, + "theoretical_loss": 3.6626679086195923, + "tokens_seen": 962134016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035783350050150455, + "loss": 2.5954, + "theoretical_loss": 3.6626439973737765, + "tokens_seen": 962199552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578234704112337, + "loss": 2.8665, + "theoretical_loss": 3.6626200882124906, + "tokens_seen": 962265088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578134403209629, + "loss": 2.7817, + "theoretical_loss": 3.6625961811354104, + "tokens_seen": 962330624 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003578034102306921, + "loss": 2.4717, + "theoretical_loss": 3.662572276142212, + "tokens_seen": 962396160 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577933801404213, + "loss": 2.6832, + "theoretical_loss": 3.662548373232572, + "tokens_seen": 962461696 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035778335005015046, + "loss": 2.5416, + "theoretical_loss": 3.662524472406168, + "tokens_seen": 962527232 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035777331995987964, + "loss": 2.7669, + "theoretical_loss": 3.662500573662675, + "tokens_seen": 962592768 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577632898696088, + "loss": 2.6586, + "theoretical_loss": 3.66247667700177, + "tokens_seen": 962658304 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035775325977933806, + "loss": 2.4959, + "theoretical_loss": 3.662452782423131, + "tokens_seen": 962723840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577432296890672, + "loss": 2.7372, + "theoretical_loss": 3.662428889926434, + "tokens_seen": 962789376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577331995987964, + "loss": 2.9573, + "theoretical_loss": 3.662404999511356, + "tokens_seen": 962854912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035772316950852555, + "loss": 2.5443, + "theoretical_loss": 3.6623811111775737, + "tokens_seen": 962920448 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003577131394182548, + "loss": 2.7655, + "theoretical_loss": 3.6623572249247642, + "tokens_seen": 962985984 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035770310932798396, + "loss": 2.6151, + "theoretical_loss": 3.662333340752604, + "tokens_seen": 963051520 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035769307923771314, + "loss": 2.7865, + "theoretical_loss": 3.662309458660772, + "tokens_seen": 963117056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576830491474423, + "loss": 2.7037, + "theoretical_loss": 3.6622855786489446, + "tokens_seen": 963182592 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035767301905717156, + "loss": 2.4568, + "theoretical_loss": 3.6622617007167992, + "tokens_seen": 963248128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576629889669007, + "loss": 2.5051, + "theoretical_loss": 3.6622378248640124, + "tokens_seen": 963313664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 538877, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.671361207962036, + "objective/train/theoretical_loss": 3.6622139510902625, + "objective/train/tokens_used": 983839200, + "theoretical_loss": 3.6622139510902625, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576529588766299, + "loss": 2.464, + "theoretical_loss": 3.6622139510902625, + "tokens_seen": 963379200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035764292878635905, + "loss": 2.8269, + "theoretical_loss": 3.662190079395227, + "tokens_seen": 963444736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003576328986960883, + "loss": 2.7938, + "theoretical_loss": 3.6621662097785834, + "tokens_seen": 963510272 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035762286860581747, + "loss": 2.6486, + "theoretical_loss": 3.66214234224001, + "tokens_seen": 963575808 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035761283851554665, + "loss": 2.713, + "theoretical_loss": 3.662118476779183, + "tokens_seen": 963641344 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035760280842527583, + "loss": 2.7186, + "theoretical_loss": 3.662094613395782, + "tokens_seen": 963706880 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357592778335005, + "loss": 2.4699, + "theoretical_loss": 3.6620707520894835, + "tokens_seen": 963772416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575827482447342, + "loss": 2.7424, + "theoretical_loss": 3.662046892859966, + "tokens_seen": 963837952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575727181544634, + "loss": 2.6724, + "theoretical_loss": 3.662023035706908, + "tokens_seen": 963903488 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035756268806419255, + "loss": 2.6567, + "theoretical_loss": 3.661999180629987, + "tokens_seen": 963969024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575526579739218, + "loss": 2.5726, + "theoretical_loss": 3.6619753276288822, + "tokens_seen": 964034560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575426278836509, + "loss": 2.6506, + "theoretical_loss": 3.661951476703271, + "tokens_seen": 964100096 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035753259779338015, + "loss": 2.6112, + "theoretical_loss": 3.6619276278528314, + "tokens_seen": 964165632 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035752256770310933, + "loss": 2.6589, + "theoretical_loss": 3.6619037810772426, + "tokens_seen": 964231168 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575125376128385, + "loss": 2.7243, + "theoretical_loss": 3.6618799363761827, + "tokens_seen": 964296704 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003575025075225677, + "loss": 2.7628, + "theoretical_loss": 3.6618560937493303, + "tokens_seen": 964362240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035749247743229693, + "loss": 2.6175, + "theoretical_loss": 3.661832253196364, + "tokens_seen": 964427776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035748244734202606, + "loss": 2.6583, + "theoretical_loss": 3.6618084147169627, + "tokens_seen": 964493312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574724172517553, + "loss": 2.9021, + "theoretical_loss": 3.6617845783108054, + "tokens_seen": 964558848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574623871614844, + "loss": 2.653, + "theoretical_loss": 3.6617607439775703, + "tokens_seen": 964624384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035745235707121365, + "loss": 2.4431, + "theoretical_loss": 3.661736911716936, + "tokens_seen": 964689920 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035744232698094283, + "loss": 2.6706, + "theoretical_loss": 3.6617130815285828, + "tokens_seen": 964755456 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357432296890672, + "loss": 2.6288, + "theoretical_loss": 3.661689253412189, + "tokens_seen": 964820992 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574222668004012, + "loss": 2.7101, + "theoretical_loss": 3.661665427367434, + "tokens_seen": 964886528 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003574122367101304, + "loss": 2.7293, + "theoretical_loss": 3.6616416033939965, + "tokens_seen": 964952064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 539426, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4708588123321533, + "objective/train/theoretical_loss": 3.6616177814915556, + "objective/train/tokens_used": 985477600, + "theoretical_loss": 3.6616177814915556, + "tokens_seen": 965017600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035740220661985956, + "loss": 2.766, + "theoretical_loss": 3.6616177814915556, + "tokens_seen": 965017600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573921765295888, + "loss": 2.9734, + "theoretical_loss": 3.6615939616597917, + "tokens_seen": 965083136 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573821464393179, + "loss": 2.5102, + "theoretical_loss": 3.661570143898383, + "tokens_seen": 965148672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035737211634904716, + "loss": 2.6679, + "theoretical_loss": 3.6615463282070104, + "tokens_seen": 965214208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573620862587763, + "loss": 2.6167, + "theoretical_loss": 3.6615225145853523, + "tokens_seen": 965279744 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573520561685055, + "loss": 2.6641, + "theoretical_loss": 3.6614987030330886, + "tokens_seen": 965345280 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035734202607823475, + "loss": 2.8822, + "theoretical_loss": 3.661474893549899, + "tokens_seen": 965410816 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573319959879639, + "loss": 2.8572, + "theoretical_loss": 3.6614510861354637, + "tokens_seen": 965476352 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573219658976931, + "loss": 2.6992, + "theoretical_loss": 3.661427280789462, + "tokens_seen": 965541888 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573119358074223, + "loss": 2.5701, + "theoretical_loss": 3.6614034775115742, + "tokens_seen": 965607424 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003573019057171515, + "loss": 2.788, + "theoretical_loss": 3.66137967630148, + "tokens_seen": 965672960 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035729187562688066, + "loss": 2.9269, + "theoretical_loss": 3.6613558771588592, + "tokens_seen": 965738496 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035728184553660984, + "loss": 2.7455, + "theoretical_loss": 3.6613320800833926, + "tokens_seen": 965804032 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357271815446339, + "loss": 2.7057, + "theoretical_loss": 3.6613082850747602, + "tokens_seen": 965869568 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035726178535606826, + "loss": 2.7845, + "theoretical_loss": 3.6612844921326415, + "tokens_seen": 965935104 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572517552657974, + "loss": 2.6283, + "theoretical_loss": 3.661260701256718, + "tokens_seen": 966000640 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003572417251755266, + "loss": 2.5378, + "theoretical_loss": 3.66123691244667, + "tokens_seen": 966066176 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035723169508525575, + "loss": 2.626, + "theoretical_loss": 3.6612131257021767, + "tokens_seen": 966131712 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357221664994985, + "loss": 2.6145, + "theoretical_loss": 3.6611893410229204, + "tokens_seen": 966197248 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035721163490471416, + "loss": 2.5952, + "theoretical_loss": 3.6611655584085803, + "tokens_seen": 966262784 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035720160481444334, + "loss": 2.3264, + "theoretical_loss": 3.6611417778588375, + "tokens_seen": 966328320 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571915747241725, + "loss": 2.5936, + "theoretical_loss": 3.661117999373374, + "tokens_seen": 966393856 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035718154463390176, + "loss": 2.7259, + "theoretical_loss": 3.661094222951869, + "tokens_seen": 966459392 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571715145436309, + "loss": 2.6951, + "theoretical_loss": 3.6610704485940033, + "tokens_seen": 966524928 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571614844533601, + "loss": 2.7819, + "theoretical_loss": 3.661046676299459, + "tokens_seen": 966590464 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 540676, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.884268283843994, + "objective/train/theoretical_loss": 3.6610229060679167, + "objective/train/tokens_used": 987116000, + "theoretical_loss": 3.6610229060679167, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035715145436308925, + "loss": 2.5583, + "theoretical_loss": 3.6610229060679167, + "tokens_seen": 966656000 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571414242728185, + "loss": 2.6991, + "theoretical_loss": 3.660999137899058, + "tokens_seen": 966721536 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035713139418254767, + "loss": 2.7206, + "theoretical_loss": 3.660975371792563, + "tokens_seen": 966787072 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035712136409227685, + "loss": 2.681, + "theoretical_loss": 3.660951607748114, + "tokens_seen": 966852608 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035711133400200603, + "loss": 2.5881, + "theoretical_loss": 3.660927845765392, + "tokens_seen": 966918144 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003571013039117352, + "loss": 2.7071, + "theoretical_loss": 3.660904085844078, + "tokens_seen": 966983680 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570912738214644, + "loss": 2.7228, + "theoretical_loss": 3.660880327983854, + "tokens_seen": 967049216 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570812437311936, + "loss": 2.6058, + "theoretical_loss": 3.660856572184401, + "tokens_seen": 967114752 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035707121364092275, + "loss": 2.5509, + "theoretical_loss": 3.6608328184454013, + "tokens_seen": 967180288 + }, + { + "epoch": 0.29, + "learning_rate": 0.000357061183550652, + "loss": 2.6556, + "theoretical_loss": 3.6608090667665363, + "tokens_seen": 967245824 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570511534603811, + "loss": 2.796, + "theoretical_loss": 3.6607853171474876, + "tokens_seen": 967311360 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035704112337011035, + "loss": 2.6946, + "theoretical_loss": 3.660761569587937, + "tokens_seen": 967376896 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035703109327983953, + "loss": 2.5267, + "theoretical_loss": 3.660737824087567, + "tokens_seen": 967442432 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570210631895687, + "loss": 2.7813, + "theoretical_loss": 3.660714080646059, + "tokens_seen": 967507968 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003570110330992979, + "loss": 2.6722, + "theoretical_loss": 3.660690339263095, + "tokens_seen": 967573504 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035700100300902713, + "loss": 2.8932, + "theoretical_loss": 3.6606665999383576, + "tokens_seen": 967639040 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035699097291875626, + "loss": 2.636, + "theoretical_loss": 3.660642862671528, + "tokens_seen": 967704576 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569809428284855, + "loss": 2.7414, + "theoretical_loss": 3.66061912746229, + "tokens_seen": 967770112 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569709127382146, + "loss": 2.823, + "theoretical_loss": 3.6605953943103247, + "tokens_seen": 967835648 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035696088264794385, + "loss": 2.8707, + "theoretical_loss": 3.6605716632153147, + "tokens_seen": 967901184 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035695085255767303, + "loss": 2.7507, + "theoretical_loss": 3.660547934176943, + "tokens_seen": 967966720 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569408224674022, + "loss": 2.9235, + "theoretical_loss": 3.660524207194891, + "tokens_seen": 968032256 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569307923771314, + "loss": 2.5484, + "theoretical_loss": 3.660500482268843, + "tokens_seen": 968097792 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003569207622868606, + "loss": 2.7966, + "theoretical_loss": 3.66047675939848, + "tokens_seen": 968163328 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035691073219658976, + "loss": 2.7508, + "theoretical_loss": 3.660453038583486, + "tokens_seen": 968228864 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 541190, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9151451587677, + "objective/train/theoretical_loss": 3.660429319823543, + "objective/train/tokens_used": 988754400, + "theoretical_loss": 3.660429319823543, + "tokens_seen": 968294400 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356900702106319, + "loss": 2.8765, + "theoretical_loss": 3.660429319823543, + "tokens_seen": 968294400 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568906720160481, + "loss": 2.9427, + "theoretical_loss": 3.6604056031183343, + "tokens_seen": 968359936 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035688064192577736, + "loss": 2.8068, + "theoretical_loss": 3.6603818884675423, + "tokens_seen": 968425472 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568706118355065, + "loss": 2.6032, + "theoretical_loss": 3.660358175870851, + "tokens_seen": 968491008 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568605817452357, + "loss": 2.5964, + "theoretical_loss": 3.660334465327943, + "tokens_seen": 968556544 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568505516549649, + "loss": 2.7074, + "theoretical_loss": 3.6603107568385016, + "tokens_seen": 968622080 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568405215646941, + "loss": 2.8676, + "theoretical_loss": 3.66028705040221, + "tokens_seen": 968687616 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035683049147442326, + "loss": 2.6544, + "theoretical_loss": 3.6602633460187506, + "tokens_seen": 968753152 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568204613841525, + "loss": 2.7562, + "theoretical_loss": 3.6602396436878077, + "tokens_seen": 968818688 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003568104312938816, + "loss": 2.3719, + "theoretical_loss": 3.6602159434090655, + "tokens_seen": 968884224 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035680040120361086, + "loss": 2.7534, + "theoretical_loss": 3.660192245182206, + "tokens_seen": 968949760 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035679037111334, + "loss": 2.534, + "theoretical_loss": 3.6601685490069134, + "tokens_seen": 969015296 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567803410230692, + "loss": 2.62, + "theoretical_loss": 3.660144854882872, + "tokens_seen": 969080832 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567703109327984, + "loss": 2.8146, + "theoretical_loss": 3.660121162809764, + "tokens_seen": 969146368 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567602808425276, + "loss": 2.7872, + "theoretical_loss": 3.6600974727872746, + "tokens_seen": 969211904 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035675025075225677, + "loss": 2.7477, + "theoretical_loss": 3.6600737848150873, + "tokens_seen": 969277440 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035674022066198595, + "loss": 2.8665, + "theoretical_loss": 3.660050098892886, + "tokens_seen": 969342976 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035673019057171513, + "loss": 2.9426, + "theoretical_loss": 3.660026415020355, + "tokens_seen": 969408512 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035672016048144436, + "loss": 2.5801, + "theoretical_loss": 3.6600027331971776, + "tokens_seen": 969474048 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567101303911735, + "loss": 2.5656, + "theoretical_loss": 3.6599790534230383, + "tokens_seen": 969539584 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003567001003009027, + "loss": 2.6174, + "theoretical_loss": 3.659955375697622, + "tokens_seen": 969605120 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035669007021063185, + "loss": 2.5564, + "theoretical_loss": 3.659931700020612, + "tokens_seen": 969670656 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003566800401203611, + "loss": 2.4917, + "theoretical_loss": 3.659908026391693, + "tokens_seen": 969736192 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035667001003009027, + "loss": 2.6846, + "theoretical_loss": 3.6598843548105497, + "tokens_seen": 969801728 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035665997993981945, + "loss": 2.7662, + "theoretical_loss": 3.6598606852768665, + "tokens_seen": 969867264 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 542499, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.745290517807007, + "objective/train/theoretical_loss": 3.659837017790328, + "objective/train/tokens_used": 990392800, + "theoretical_loss": 3.659837017790328, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035664994984954863, + "loss": 2.8151, + "theoretical_loss": 3.659837017790328, + "tokens_seen": 969932800 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035663991975927787, + "loss": 2.6502, + "theoretical_loss": 3.6598133523506187, + "tokens_seen": 969998336 + }, + { + "epoch": 0.29, + "learning_rate": 0.000356629889669007, + "loss": 2.6904, + "theoretical_loss": 3.6597896889574235, + "tokens_seen": 970063872 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035661985957873623, + "loss": 2.7728, + "theoretical_loss": 3.6597660276104262, + "tokens_seen": 970129408 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035660982948846536, + "loss": 2.4748, + "theoretical_loss": 3.6597423683093133, + "tokens_seen": 970194944 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565997993981946, + "loss": 2.4516, + "theoretical_loss": 3.6597187110537686, + "tokens_seen": 970260480 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565897693079238, + "loss": 3.0593, + "theoretical_loss": 3.6596950558434775, + "tokens_seen": 970326016 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035657973921765295, + "loss": 2.6859, + "theoretical_loss": 3.659671402678125, + "tokens_seen": 970391552 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565697091273822, + "loss": 2.7334, + "theoretical_loss": 3.6596477515573964, + "tokens_seen": 970457088 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565596790371113, + "loss": 2.5428, + "theoretical_loss": 3.6596241024809766, + "tokens_seen": 970522624 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035654964894684055, + "loss": 2.6518, + "theoretical_loss": 3.65960045544855, + "tokens_seen": 970588160 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035653961885656973, + "loss": 2.608, + "theoretical_loss": 3.6595768104598045, + "tokens_seen": 970653696 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565295887662989, + "loss": 2.776, + "theoretical_loss": 3.659553167514423, + "tokens_seen": 970719232 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003565195586760281, + "loss": 2.8407, + "theoretical_loss": 3.659529526612092, + "tokens_seen": 970784768 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035650952858575733, + "loss": 2.5555, + "theoretical_loss": 3.659505887752497, + "tokens_seen": 970850304 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035649949849548646, + "loss": 2.8917, + "theoretical_loss": 3.6594822509353238, + "tokens_seen": 970915840 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564894684052157, + "loss": 2.8816, + "theoretical_loss": 3.659458616160258, + "tokens_seen": 970981376 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564794383149448, + "loss": 2.5581, + "theoretical_loss": 3.6594349834269844, + "tokens_seen": 971046912 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035646940822467405, + "loss": 2.3331, + "theoretical_loss": 3.65941135273519, + "tokens_seen": 971112448 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035645937813440323, + "loss": 2.658, + "theoretical_loss": 3.6593877240845605, + "tokens_seen": 971177984 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564493480441324, + "loss": 2.8178, + "theoretical_loss": 3.659364097474782, + "tokens_seen": 971243520 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564393179538616, + "loss": 2.5768, + "theoretical_loss": 3.65934047290554, + "tokens_seen": 971309056 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564292878635908, + "loss": 2.509, + "theoretical_loss": 3.6593168503765203, + "tokens_seen": 971374592 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035641925777331996, + "loss": 2.5761, + "theoretical_loss": 3.6592932298874103, + "tokens_seen": 971440128 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003564092276830492, + "loss": 2.8889, + "theoretical_loss": 3.659269611437895, + "tokens_seen": 971505664 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 543787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6714510917663574, + "objective/train/theoretical_loss": 3.6592459950276623, + "objective/train/tokens_used": 992031200, + "theoretical_loss": 3.6592459950276623, + "tokens_seen": 971571200 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563991975927783, + "loss": 2.7689, + "theoretical_loss": 3.6592459950276623, + "tokens_seen": 971571200 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035638916750250756, + "loss": 2.3759, + "theoretical_loss": 3.659222380656397, + "tokens_seen": 971636736 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563791374122367, + "loss": 2.6477, + "theoretical_loss": 3.659198768323786, + "tokens_seen": 971702272 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563691073219659, + "loss": 2.633, + "theoretical_loss": 3.659175158029516, + "tokens_seen": 971767808 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563590772316951, + "loss": 2.8631, + "theoretical_loss": 3.6591515497732736, + "tokens_seen": 971833344 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563490471414243, + "loss": 2.9037, + "theoretical_loss": 3.6591279435547452, + "tokens_seen": 971898880 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035633901705115346, + "loss": 2.6792, + "theoretical_loss": 3.6591043393736173, + "tokens_seen": 971964416 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563289869608827, + "loss": 2.6147, + "theoretical_loss": 3.659080737229578, + "tokens_seen": 972029952 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003563189568706118, + "loss": 2.5756, + "theoretical_loss": 3.6590571371223133, + "tokens_seen": 972095488 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035630892678034106, + "loss": 2.5859, + "theoretical_loss": 3.6590335390515096, + "tokens_seen": 972161024 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562988966900702, + "loss": 2.7238, + "theoretical_loss": 3.6590099430168546, + "tokens_seen": 972226560 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562888665997994, + "loss": 2.7529, + "theoretical_loss": 3.6589863490180354, + "tokens_seen": 972292096 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562788365095286, + "loss": 2.6947, + "theoretical_loss": 3.6589627570547387, + "tokens_seen": 972357632 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562688064192578, + "loss": 2.8994, + "theoretical_loss": 3.6589391671266522, + "tokens_seen": 972423168 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035625877632898697, + "loss": 2.6734, + "theoretical_loss": 3.6589155792334633, + "tokens_seen": 972488704 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035624874623871615, + "loss": 2.7424, + "theoretical_loss": 3.6588919933748585, + "tokens_seen": 972554240 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035623871614844533, + "loss": 2.8223, + "theoretical_loss": 3.658868409550526, + "tokens_seen": 972619776 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035622868605817456, + "loss": 2.7093, + "theoretical_loss": 3.6588448277601526, + "tokens_seen": 972685312 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562186559679037, + "loss": 2.7391, + "theoretical_loss": 3.658821248003427, + "tokens_seen": 972750848 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003562086258776329, + "loss": 2.7881, + "theoretical_loss": 3.6587976702800353, + "tokens_seen": 972816384 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035619859578736205, + "loss": 2.6558, + "theoretical_loss": 3.658774094589667, + "tokens_seen": 972881920 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561885656970913, + "loss": 2.6641, + "theoretical_loss": 3.658750520932008, + "tokens_seen": 972947456 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035617853560682047, + "loss": 2.6948, + "theoretical_loss": 3.6587269493067476, + "tokens_seen": 973012992 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035616850551654965, + "loss": 2.6006, + "theoretical_loss": 3.6587033797135726, + "tokens_seen": 973078528 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035615847542627883, + "loss": 2.5074, + "theoretical_loss": 3.658679812152172, + "tokens_seen": 973144064 + }, + { + "epoch": 0.29, + "objective/train/docs_used": 544242, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9670348167419434, + "objective/train/theoretical_loss": 3.658656246622233, + "objective/train/tokens_used": 993669600, + "theoretical_loss": 3.658656246622233, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035614844533600807, + "loss": 2.8928, + "theoretical_loss": 3.658656246622233, + "tokens_seen": 973209600 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561384152457372, + "loss": 2.6655, + "theoretical_loss": 3.658632683123444, + "tokens_seen": 973275136 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035612838515546643, + "loss": 2.6035, + "theoretical_loss": 3.658609121655494, + "tokens_seen": 973340672 + }, + { + "epoch": 0.29, + "learning_rate": 0.00035611835506519556, + "loss": 2.646, + "theoretical_loss": 3.65858556221807, + "tokens_seen": 973406208 + }, + { + "epoch": 0.29, + "learning_rate": 0.0003561083249749248, + "loss": 2.6821, + "theoretical_loss": 3.6585620048108605, + "tokens_seen": 973471744 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035609829488465397, + "loss": 2.8966, + "theoretical_loss": 3.6585384494335544, + "tokens_seen": 973537280 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035608826479438315, + "loss": 2.6911, + "theoretical_loss": 3.65851489608584, + "tokens_seen": 973602816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035607823470411233, + "loss": 2.7678, + "theoretical_loss": 3.658491344767406, + "tokens_seen": 973668352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560682046138415, + "loss": 2.8951, + "theoretical_loss": 3.6584677954779403, + "tokens_seen": 973733888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560581745235707, + "loss": 2.68, + "theoretical_loss": 3.6584442482171324, + "tokens_seen": 973799424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035604814443329993, + "loss": 2.8754, + "theoretical_loss": 3.658420702984671, + "tokens_seen": 973864960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035603811434302906, + "loss": 2.6871, + "theoretical_loss": 3.658397159780244, + "tokens_seen": 973930496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560280842527583, + "loss": 2.4913, + "theoretical_loss": 3.658373618603541, + "tokens_seen": 973996032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003560180541624874, + "loss": 2.5447, + "theoretical_loss": 3.6583500794542503, + "tokens_seen": 974061568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035600802407221666, + "loss": 2.6892, + "theoretical_loss": 3.6583265423320617, + "tokens_seen": 974127104 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035599799398194584, + "loss": 2.5806, + "theoretical_loss": 3.6583030072366647, + "tokens_seen": 974192640 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355987963891675, + "loss": 2.5213, + "theoretical_loss": 3.6582794741677467, + "tokens_seen": 974258176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559779338014042, + "loss": 2.6246, + "theoretical_loss": 3.658255943124998, + "tokens_seen": 974323712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035596790371113344, + "loss": 2.7462, + "theoretical_loss": 3.6582324141081086, + "tokens_seen": 974389248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035595787362086256, + "loss": 2.6027, + "theoretical_loss": 3.6582088871167664, + "tokens_seen": 974454784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559478435305918, + "loss": 2.6232, + "theoretical_loss": 3.6581853621506615, + "tokens_seen": 974520320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559378134403209, + "loss": 2.7216, + "theoretical_loss": 3.658161839209484, + "tokens_seen": 974585856 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035592778335005016, + "loss": 2.8452, + "theoretical_loss": 3.658138318292922, + "tokens_seen": 974651392 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035591775325977934, + "loss": 2.6781, + "theoretical_loss": 3.658114799400666, + "tokens_seen": 974716928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003559077231695085, + "loss": 2.5703, + "theoretical_loss": 3.6580912825324057, + "tokens_seen": 974782464 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 545526, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5406079292297363, + "objective/train/theoretical_loss": 3.6580677676878306, + "objective/train/tokens_used": 995308000, + "theoretical_loss": 3.6580677676878306, + "tokens_seen": 974848000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558976930792377, + "loss": 2.6219, + "theoretical_loss": 3.6580677676878306, + "tokens_seen": 974848000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558876629889669, + "loss": 2.7312, + "theoretical_loss": 3.658044254866631, + "tokens_seen": 974913536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035587763289869607, + "loss": 2.6663, + "theoretical_loss": 3.6580207440684966, + "tokens_seen": 974979072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558676028084253, + "loss": 2.7667, + "theoretical_loss": 3.657997235293117, + "tokens_seen": 975044608 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035585757271815443, + "loss": 2.7782, + "theoretical_loss": 3.6579737285401825, + "tokens_seen": 975110144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035584754262788366, + "loss": 2.3444, + "theoretical_loss": 3.6579502238093835, + "tokens_seen": 975175680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558375125376129, + "loss": 2.7807, + "theoretical_loss": 3.6579267211004094, + "tokens_seen": 975241216 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355827482447342, + "loss": 2.7576, + "theoretical_loss": 3.657903220412951, + "tokens_seen": 975306752 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035581745235707126, + "loss": 2.8424, + "theoretical_loss": 3.657879721746699, + "tokens_seen": 975372288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003558074222668004, + "loss": 2.7173, + "theoretical_loss": 3.657856225101343, + "tokens_seen": 975437824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557973921765296, + "loss": 2.7882, + "theoretical_loss": 3.6578327304765734, + "tokens_seen": 975503360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557873620862588, + "loss": 2.7136, + "theoretical_loss": 3.6578092378720815, + "tokens_seen": 975568896 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355777331995988, + "loss": 2.7584, + "theoretical_loss": 3.6577857472875572, + "tokens_seen": 975634432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035576730190571717, + "loss": 2.6165, + "theoretical_loss": 3.6577622587226917, + "tokens_seen": 975699968 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035575727181544635, + "loss": 2.7483, + "theoretical_loss": 3.657738772177175, + "tokens_seen": 975765504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035574724172517553, + "loss": 2.5081, + "theoretical_loss": 3.6577152876506984, + "tokens_seen": 975831040 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035573721163490476, + "loss": 3.0255, + "theoretical_loss": 3.6576918051429526, + "tokens_seen": 975896576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557271815446339, + "loss": 2.4618, + "theoretical_loss": 3.6576683246536286, + "tokens_seen": 975962112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003557171514543631, + "loss": 2.5939, + "theoretical_loss": 3.657644846182417, + "tokens_seen": 976027648 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035570712136409225, + "loss": 3.0158, + "theoretical_loss": 3.65762136972901, + "tokens_seen": 976093184 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556970912738215, + "loss": 2.6443, + "theoretical_loss": 3.657597895293097, + "tokens_seen": 976158720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035568706118355067, + "loss": 2.648, + "theoretical_loss": 3.6575744228743705, + "tokens_seen": 976224256 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035567703109327985, + "loss": 2.5602, + "theoretical_loss": 3.6575509524725214, + "tokens_seen": 976289792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035566700100300903, + "loss": 2.8696, + "theoretical_loss": 3.657527484087241, + "tokens_seen": 976355328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035565697091273827, + "loss": 2.7943, + "theoretical_loss": 3.6575040177182205, + "tokens_seen": 976420864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 546148, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.82014799118042, + "objective/train/theoretical_loss": 3.6574805533651515, + "objective/train/tokens_used": 996946400, + "theoretical_loss": 3.6574805533651515, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003556469408224674, + "loss": 2.6284, + "theoretical_loss": 3.6574805533651515, + "tokens_seen": 976486400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035563691073219663, + "loss": 2.7933, + "theoretical_loss": 3.6574570910277258, + "tokens_seen": 976551936 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035562688064192576, + "loss": 2.8423, + "theoretical_loss": 3.6574336307056345, + "tokens_seen": 976617472 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355616850551655, + "loss": 2.6661, + "theoretical_loss": 3.65741017239857, + "tokens_seen": 976683008 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035560682046138417, + "loss": 2.6578, + "theoretical_loss": 3.6573867161062235, + "tokens_seen": 976748544 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035559679037111335, + "loss": 2.6173, + "theoretical_loss": 3.6573632618282867, + "tokens_seen": 976814080 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035558676028084253, + "loss": 2.6873, + "theoretical_loss": 3.6573398095644514, + "tokens_seen": 976879616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555767301905717, + "loss": 2.5844, + "theoretical_loss": 3.6573163593144105, + "tokens_seen": 976945152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555667001003009, + "loss": 2.7901, + "theoretical_loss": 3.657292911077855, + "tokens_seen": 977010688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035555667001003013, + "loss": 2.47, + "theoretical_loss": 3.657269464854477, + "tokens_seen": 977076224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035554663991975926, + "loss": 2.576, + "theoretical_loss": 3.657246020643969, + "tokens_seen": 977141760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555366098294885, + "loss": 2.7507, + "theoretical_loss": 3.6572225784460235, + "tokens_seen": 977207296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003555265797392176, + "loss": 2.6684, + "theoretical_loss": 3.6571991382603324, + "tokens_seen": 977272832 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035551654964894686, + "loss": 2.6614, + "theoretical_loss": 3.6571757000865874, + "tokens_seen": 977338368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035550651955867604, + "loss": 2.8629, + "theoretical_loss": 3.6571522639244822, + "tokens_seen": 977403904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554964894684052, + "loss": 2.6595, + "theoretical_loss": 3.6571288297737086, + "tokens_seen": 977469440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554864593781344, + "loss": 2.6743, + "theoretical_loss": 3.657105397633959, + "tokens_seen": 977534976 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035547642928786364, + "loss": 2.7305, + "theoretical_loss": 3.657081967504926, + "tokens_seen": 977600512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035546639919759276, + "loss": 2.7035, + "theoretical_loss": 3.657058539386303, + "tokens_seen": 977666048 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355456369107322, + "loss": 2.7986, + "theoretical_loss": 3.6570351132777823, + "tokens_seen": 977731584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554463390170511, + "loss": 2.6578, + "theoretical_loss": 3.657011689179056, + "tokens_seen": 977797120 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035543630892678036, + "loss": 2.6389, + "theoretical_loss": 3.656988267089818, + "tokens_seen": 977862656 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035542627883650954, + "loss": 2.5991, + "theoretical_loss": 3.6569648470097613, + "tokens_seen": 977928192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554162487462387, + "loss": 2.7215, + "theoretical_loss": 3.6569414289385778, + "tokens_seen": 977993728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003554062186559679, + "loss": 2.5016, + "theoretical_loss": 3.6569180128759617, + "tokens_seen": 978059264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 547377, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.918895721435547, + "objective/train/theoretical_loss": 3.656894598821606, + "objective/train/tokens_used": 998584800, + "theoretical_loss": 3.656894598821606, + "tokens_seen": 978124800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553961885656971, + "loss": 2.7372, + "theoretical_loss": 3.656894598821606, + "tokens_seen": 978124800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035538615847542627, + "loss": 2.6627, + "theoretical_loss": 3.6568711867752035, + "tokens_seen": 978190336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553761283851555, + "loss": 2.7191, + "theoretical_loss": 3.6568477767364476, + "tokens_seen": 978255872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035536609829488463, + "loss": 2.7288, + "theoretical_loss": 3.656824368705032, + "tokens_seen": 978321408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035535606820461386, + "loss": 2.7022, + "theoretical_loss": 3.6568009626806495, + "tokens_seen": 978386944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035534603811434304, + "loss": 2.6384, + "theoretical_loss": 3.6567775586629945, + "tokens_seen": 978452480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553360080240722, + "loss": 2.8189, + "theoretical_loss": 3.6567541566517594, + "tokens_seen": 978518016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553259779338014, + "loss": 2.7914, + "theoretical_loss": 3.6567307566466387, + "tokens_seen": 978583552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003553159478435306, + "loss": 2.8269, + "theoretical_loss": 3.656707358647326, + "tokens_seen": 978649088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035530591775325977, + "loss": 2.4179, + "theoretical_loss": 3.6566839626535153, + "tokens_seen": 978714624 + }, + { + "epoch": 0.3, + "learning_rate": 0.000355295887662989, + "loss": 2.6729, + "theoretical_loss": 3.6566605686648996, + "tokens_seen": 978780160 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035528585757271813, + "loss": 2.6041, + "theoretical_loss": 3.6566371766811727, + "tokens_seen": 978845696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035527582748244737, + "loss": 2.5674, + "theoretical_loss": 3.65661378670203, + "tokens_seen": 978911232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552657973921765, + "loss": 2.6614, + "theoretical_loss": 3.6565903987271637, + "tokens_seen": 978976768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035525576730190573, + "loss": 2.8475, + "theoretical_loss": 3.6565670127562697, + "tokens_seen": 979042304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552457372116349, + "loss": 2.7853, + "theoretical_loss": 3.656543628789041, + "tokens_seen": 979107840 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003552357071213641, + "loss": 2.7585, + "theoretical_loss": 3.6565202468251723, + "tokens_seen": 979173376 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035522567703109327, + "loss": 2.7385, + "theoretical_loss": 3.6564968668643574, + "tokens_seen": 979238912 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035521564694082245, + "loss": 2.7629, + "theoretical_loss": 3.656473488906291, + "tokens_seen": 979304448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035520561685055163, + "loss": 2.501, + "theoretical_loss": 3.656450112950668, + "tokens_seen": 979369984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035519558676028087, + "loss": 2.6088, + "theoretical_loss": 3.6564267389971823, + "tokens_seen": 979435520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035518555667001, + "loss": 2.4397, + "theoretical_loss": 3.6564033670455283, + "tokens_seen": 979501056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035517552657973923, + "loss": 2.6795, + "theoretical_loss": 3.656379997095401, + "tokens_seen": 979566592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551654964894684, + "loss": 2.5704, + "theoretical_loss": 3.6563566291464955, + "tokens_seen": 979632128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551554663991976, + "loss": 2.7986, + "theoretical_loss": 3.6563332631985053, + "tokens_seen": 979697664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 547866, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.091559648513794, + "objective/train/theoretical_loss": 3.6563098992511267, + "objective/train/tokens_used": 1000223200, + "theoretical_loss": 3.6563098992511267, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551454363089268, + "loss": 2.9244, + "theoretical_loss": 3.6563098992511267, + "tokens_seen": 979763200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035513540621865596, + "loss": 2.7605, + "theoretical_loss": 3.656286537304054, + "tokens_seen": 979828736 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035512537612838514, + "loss": 3.0127, + "theoretical_loss": 3.6562631773569816, + "tokens_seen": 979894272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035511534603811437, + "loss": 2.7358, + "theoretical_loss": 3.6562398194096053, + "tokens_seen": 979959808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003551053159478435, + "loss": 2.5052, + "theoretical_loss": 3.65621646346162, + "tokens_seen": 980025344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035509528585757273, + "loss": 2.6234, + "theoretical_loss": 3.6561931095127207, + "tokens_seen": 980090880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550852557673019, + "loss": 2.7937, + "theoretical_loss": 3.656169757562603, + "tokens_seen": 980156416 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550752256770311, + "loss": 2.7786, + "theoretical_loss": 3.6561464076109615, + "tokens_seen": 980221952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035506519558676033, + "loss": 2.5644, + "theoretical_loss": 3.6561230596574927, + "tokens_seen": 980287488 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035505516549648946, + "loss": 2.791, + "theoretical_loss": 3.656099713701891, + "tokens_seen": 980353024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550451354062187, + "loss": 2.7434, + "theoretical_loss": 3.6560763697438525, + "tokens_seen": 980418560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550351053159478, + "loss": 2.6858, + "theoretical_loss": 3.6560530277830723, + "tokens_seen": 980484096 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035502507522567706, + "loss": 2.6007, + "theoretical_loss": 3.656029687819246, + "tokens_seen": 980549632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035501504513540624, + "loss": 2.7632, + "theoretical_loss": 3.6560063498520705, + "tokens_seen": 980615168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003550050150451354, + "loss": 2.6044, + "theoretical_loss": 3.65598301388124, + "tokens_seen": 980680704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549949849548646, + "loss": 2.7278, + "theoretical_loss": 3.6559596799064513, + "tokens_seen": 980746240 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035498495486459384, + "loss": 2.6622, + "theoretical_loss": 3.6559363479274003, + "tokens_seen": 980811776 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035497492477432296, + "loss": 2.6397, + "theoretical_loss": 3.6559130179437824, + "tokens_seen": 980877312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549648946840522, + "loss": 2.4674, + "theoretical_loss": 3.655889689955294, + "tokens_seen": 980942848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549548645937813, + "loss": 2.5841, + "theoretical_loss": 3.6558663639616307, + "tokens_seen": 981008384 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035494483450351056, + "loss": 2.803, + "theoretical_loss": 3.6558430399624893, + "tokens_seen": 981073920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035493480441323974, + "loss": 2.8485, + "theoretical_loss": 3.655819717957566, + "tokens_seen": 981139456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549247743229689, + "loss": 2.7757, + "theoretical_loss": 3.655796397946557, + "tokens_seen": 981204992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549147442326981, + "loss": 2.6201, + "theoretical_loss": 3.6557730799291583, + "tokens_seen": 981270528 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003549047141424273, + "loss": 2.6034, + "theoretical_loss": 3.655749763905067, + "tokens_seen": 981336064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 549153, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9054205417633057, + "objective/train/theoretical_loss": 3.655726449873979, + "objective/train/tokens_used": 1001861600, + "theoretical_loss": 3.655726449873979, + "tokens_seen": 981401600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035489468405215647, + "loss": 2.9853, + "theoretical_loss": 3.655726449873979, + "tokens_seen": 981401600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548846539618857, + "loss": 2.9481, + "theoretical_loss": 3.6557031378355913, + "tokens_seen": 981467136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035487462387161483, + "loss": 2.6993, + "theoretical_loss": 3.6556798277896005, + "tokens_seen": 981532672 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035486459378134406, + "loss": 2.8664, + "theoretical_loss": 3.655656519735703, + "tokens_seen": 981598208 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035485456369107324, + "loss": 2.7443, + "theoretical_loss": 3.655633213673596, + "tokens_seen": 981663744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548445336008024, + "loss": 2.4849, + "theoretical_loss": 3.6556099096029757, + "tokens_seen": 981729280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548345035105316, + "loss": 2.8908, + "theoretical_loss": 3.6555866075235395, + "tokens_seen": 981794816 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548244734202608, + "loss": 2.4488, + "theoretical_loss": 3.655563307434984, + "tokens_seen": 981860352 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035481444332998997, + "loss": 2.7768, + "theoretical_loss": 3.655540009337007, + "tokens_seen": 981925888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003548044132397192, + "loss": 2.7123, + "theoretical_loss": 3.6555167132293045, + "tokens_seen": 981991424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035479438314944833, + "loss": 2.587, + "theoretical_loss": 3.6554934191115747, + "tokens_seen": 982056960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035478435305917757, + "loss": 2.6422, + "theoretical_loss": 3.6554701269835146, + "tokens_seen": 982122496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547743229689067, + "loss": 2.6564, + "theoretical_loss": 3.6554468368448205, + "tokens_seen": 982188032 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035476429287863593, + "loss": 2.6752, + "theoretical_loss": 3.6554235486951914, + "tokens_seen": 982253568 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547542627883651, + "loss": 2.6921, + "theoretical_loss": 3.6554002625343234, + "tokens_seen": 982319104 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003547442326980943, + "loss": 2.7904, + "theoretical_loss": 3.6553769783619146, + "tokens_seen": 982384640 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035473420260782347, + "loss": 2.7994, + "theoretical_loss": 3.6553536961776625, + "tokens_seen": 982450176 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035472417251755265, + "loss": 2.6377, + "theoretical_loss": 3.6553304159812647, + "tokens_seen": 982515712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035471414242728183, + "loss": 2.8887, + "theoretical_loss": 3.655307137772419, + "tokens_seen": 982581248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035470411233701107, + "loss": 2.7482, + "theoretical_loss": 3.655283861550823, + "tokens_seen": 982646784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546940822467402, + "loss": 2.5649, + "theoretical_loss": 3.6552605873161745, + "tokens_seen": 982712320 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035468405215646943, + "loss": 2.7879, + "theoretical_loss": 3.655237315068171, + "tokens_seen": 982777856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546740220661986, + "loss": 2.8483, + "theoretical_loss": 3.655214044806512, + "tokens_seen": 982843392 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546639919759278, + "loss": 2.7856, + "theoretical_loss": 3.6551907765308935, + "tokens_seen": 982908928 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354653961885657, + "loss": 2.7548, + "theoretical_loss": 3.655167510241015, + "tokens_seen": 982974464 + }, + { + "debugging/Self-BLEU-5": 0.5000497629393682, + "debugging/distinct-1-grams": 0.7839180687368331, + "debugging/distinct-2-grams": 0.9723027630693535, + "debugging/entropy-1-grams": 5.462779182741081, + "debugging/entropy-2-grams": 6.200783858933273, + "debugging/length": 666.1666666666666, + "debugging/num_segments": 6, + "epoch": 0.3, + "objective/train/docs_used": 549695, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.27803897857666, + "objective/train/theoretical_loss": 3.655144245936574, + "objective/train/tokens_used": 1003500000, + "theoretical_loss": 3.655144245936574, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035464393179538616, + "loss": 2.4059, + "theoretical_loss": 3.655144245936574, + "tokens_seen": 983040000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035463390170511534, + "loss": 2.8016, + "theoretical_loss": 3.6551209836172696, + "tokens_seen": 983105536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035462387161484457, + "loss": 2.5838, + "theoretical_loss": 3.6550977232827986, + "tokens_seen": 983171072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003546138415245737, + "loss": 2.5899, + "theoretical_loss": 3.65507446493286, + "tokens_seen": 983236608 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035460381143430293, + "loss": 2.74, + "theoretical_loss": 3.655051208567153, + "tokens_seen": 983302144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035459378134403206, + "loss": 2.8234, + "theoretical_loss": 3.655027954185376, + "tokens_seen": 983367680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545837512537613, + "loss": 2.7215, + "theoretical_loss": 3.6550047017872265, + "tokens_seen": 983433216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545737211634905, + "loss": 2.6825, + "theoretical_loss": 3.6549814513724037, + "tokens_seen": 983498752 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035456369107321966, + "loss": 2.6115, + "theoretical_loss": 3.6549582029406062, + "tokens_seen": 983564288 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035455366098294884, + "loss": 2.7599, + "theoretical_loss": 3.654934956491533, + "tokens_seen": 983629824 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354543630892678, + "loss": 2.7921, + "theoretical_loss": 3.654911712024883, + "tokens_seen": 983695360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545336008024072, + "loss": 2.7611, + "theoretical_loss": 3.6548884695403543, + "tokens_seen": 983760896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035452357071213644, + "loss": 2.7598, + "theoretical_loss": 3.6548652290376467, + "tokens_seen": 983826432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035451354062186556, + "loss": 2.7005, + "theoretical_loss": 3.654841990516459, + "tokens_seen": 983891968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003545035105315948, + "loss": 2.5206, + "theoretical_loss": 3.65481875397649, + "tokens_seen": 983957504 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354493480441324, + "loss": 2.5426, + "theoretical_loss": 3.654795519417439, + "tokens_seen": 984023040 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035448345035105316, + "loss": 2.8029, + "theoretical_loss": 3.6547722868390053, + "tokens_seen": 984088576 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035447342026078234, + "loss": 2.813, + "theoretical_loss": 3.654749056240888, + "tokens_seen": 984154112 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544633901705115, + "loss": 2.5548, + "theoretical_loss": 3.6547258276227863, + "tokens_seen": 984219648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544533600802407, + "loss": 2.6819, + "theoretical_loss": 3.6547026009844004, + "tokens_seen": 984285184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035444332998996994, + "loss": 2.5199, + "theoretical_loss": 3.6546793763254293, + "tokens_seen": 984350720 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035443329989969907, + "loss": 2.7544, + "theoretical_loss": 3.6546561536455724, + "tokens_seen": 984416256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003544232698094283, + "loss": 2.6461, + "theoretical_loss": 3.654632932944529, + "tokens_seen": 984481792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035441323971915743, + "loss": 2.6753, + "theoretical_loss": 3.654609714222, + "tokens_seen": 984547328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035440320962888667, + "loss": 2.6239, + "theoretical_loss": 3.654586497477684, + "tokens_seen": 984612864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 550994, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.944199562072754, + "objective/train/theoretical_loss": 3.6545632827112806, + "objective/train/tokens_used": 1005138400, + "theoretical_loss": 3.6545632827112806, + "tokens_seen": 984678400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035439317953861585, + "loss": 2.7087, + "theoretical_loss": 3.6545632827112806, + "tokens_seen": 984678400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035438314944834503, + "loss": 2.7407, + "theoretical_loss": 3.654540069922491, + "tokens_seen": 984743936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543731193580742, + "loss": 2.7961, + "theoretical_loss": 3.6545168591110135, + "tokens_seen": 984809472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035436308926780344, + "loss": 2.5277, + "theoretical_loss": 3.6544936502765495, + "tokens_seen": 984875008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543530591775326, + "loss": 2.5603, + "theoretical_loss": 3.6544704434187985, + "tokens_seen": 984940544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543430290872618, + "loss": 2.4406, + "theoretical_loss": 3.6544472385374602, + "tokens_seen": 985006080 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354332998996991, + "loss": 2.9052, + "theoretical_loss": 3.654424035632236, + "tokens_seen": 985071616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035432296890672017, + "loss": 2.5012, + "theoretical_loss": 3.654400834702825, + "tokens_seen": 985137152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003543129388164494, + "loss": 2.8039, + "theoretical_loss": 3.6543776357489275, + "tokens_seen": 985202688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035430290872617853, + "loss": 2.7412, + "theoretical_loss": 3.6543544387702447, + "tokens_seen": 985268224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035429287863590777, + "loss": 2.9997, + "theoretical_loss": 3.654331243766477, + "tokens_seen": 985333760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542828485456369, + "loss": 2.8078, + "theoretical_loss": 3.654308050737324, + "tokens_seen": 985399296 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035427281845536613, + "loss": 2.6121, + "theoretical_loss": 3.6542848596824875, + "tokens_seen": 985464832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542627883650953, + "loss": 2.5112, + "theoretical_loss": 3.6542616706016675, + "tokens_seen": 985530368 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542527582748245, + "loss": 2.4367, + "theoretical_loss": 3.6542384834945643, + "tokens_seen": 985595904 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035424272818455367, + "loss": 2.6558, + "theoretical_loss": 3.654215298360879, + "tokens_seen": 985661440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035423269809428285, + "loss": 2.5777, + "theoretical_loss": 3.6541921152003134, + "tokens_seen": 985726976 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035422266800401203, + "loss": 2.7154, + "theoretical_loss": 3.654168934012567, + "tokens_seen": 985792512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035421263791374127, + "loss": 2.471, + "theoretical_loss": 3.6541457547973417, + "tokens_seen": 985858048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003542026078234704, + "loss": 2.8969, + "theoretical_loss": 3.654122577554338, + "tokens_seen": 985923584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035419257773319963, + "loss": 2.5885, + "theoretical_loss": 3.6540994022832574, + "tokens_seen": 985989120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541825476429288, + "loss": 2.7422, + "theoretical_loss": 3.6540762289838007, + "tokens_seen": 986054656 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354172517552658, + "loss": 2.8467, + "theoretical_loss": 3.65405305765567, + "tokens_seen": 986120192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541624874623872, + "loss": 2.7443, + "theoretical_loss": 3.654029888298566, + "tokens_seen": 986185728 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035415245737211636, + "loss": 2.5481, + "theoretical_loss": 3.6540067209121894, + "tokens_seen": 986251264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 551567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.91987681388855, + "objective/train/theoretical_loss": 3.653983555496242, + "objective/train/tokens_used": 1006776800, + "theoretical_loss": 3.653983555496242, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035414242728184554, + "loss": 2.6924, + "theoretical_loss": 3.653983555496242, + "tokens_seen": 986316800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035413239719157477, + "loss": 2.7576, + "theoretical_loss": 3.6539603920504264, + "tokens_seen": 986382336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003541223671013039, + "loss": 2.6437, + "theoretical_loss": 3.653937230574443, + "tokens_seen": 986447872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035411233701103314, + "loss": 2.4609, + "theoretical_loss": 3.653914071067994, + "tokens_seen": 986513408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035410230692076226, + "loss": 2.7274, + "theoretical_loss": 3.6538909135307813, + "tokens_seen": 986578944 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540922768304915, + "loss": 2.7185, + "theoretical_loss": 3.653867757962505, + "tokens_seen": 986644480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540822467402207, + "loss": 2.4144, + "theoretical_loss": 3.6538446043628694, + "tokens_seen": 986710016 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035407221664994986, + "loss": 2.9986, + "theoretical_loss": 3.6538214527315747, + "tokens_seen": 986775552 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035406218655967904, + "loss": 2.7329, + "theoretical_loss": 3.6537983030683234, + "tokens_seen": 986841088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540521564694082, + "loss": 2.5728, + "theoretical_loss": 3.6537751553728173, + "tokens_seen": 986906624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540421263791374, + "loss": 2.6988, + "theoretical_loss": 3.6537520096447587, + "tokens_seen": 986972160 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035403209628886664, + "loss": 2.8755, + "theoretical_loss": 3.65372886588385, + "tokens_seen": 987037696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035402206619859577, + "loss": 2.4468, + "theoretical_loss": 3.653705724089793, + "tokens_seen": 987103232 + }, + { + "epoch": 0.3, + "learning_rate": 0.000354012036108325, + "loss": 2.6212, + "theoretical_loss": 3.65368258426229, + "tokens_seen": 987168768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003540020060180542, + "loss": 2.5444, + "theoretical_loss": 3.6536594464010435, + "tokens_seen": 987234304 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035399197592778336, + "loss": 2.7203, + "theoretical_loss": 3.653636310505756, + "tokens_seen": 987299840 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035398194583751254, + "loss": 2.7516, + "theoretical_loss": 3.6536131765761297, + "tokens_seen": 987365376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539719157472417, + "loss": 2.6541, + "theoretical_loss": 3.653590044611867, + "tokens_seen": 987430912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539618856569709, + "loss": 2.6099, + "theoretical_loss": 3.653566914612671, + "tokens_seen": 987496448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035395185556670014, + "loss": 2.7547, + "theoretical_loss": 3.653543786578244, + "tokens_seen": 987561984 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035394182547642927, + "loss": 2.8886, + "theoretical_loss": 3.6535206605082893, + "tokens_seen": 987627520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003539317953861585, + "loss": 2.7337, + "theoretical_loss": 3.653497536402509, + "tokens_seen": 987693056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035392176529588763, + "loss": 2.7313, + "theoretical_loss": 3.6534744142606064, + "tokens_seen": 987758592 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035391173520561687, + "loss": 2.7238, + "theoretical_loss": 3.653451294082284, + "tokens_seen": 987824128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035390170511534605, + "loss": 2.7589, + "theoretical_loss": 3.653428175867245, + "tokens_seen": 987889664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 552650, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.231137752532959, + "objective/train/theoretical_loss": 3.653405059615193, + "objective/train/tokens_used": 1008415200, + "theoretical_loss": 3.653405059615193, + "tokens_seen": 987955200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035389167502507523, + "loss": 2.8518, + "theoretical_loss": 3.653405059615193, + "tokens_seen": 987955200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538816449348044, + "loss": 2.7992, + "theoretical_loss": 3.65338194532583, + "tokens_seen": 988020736 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035387161484453364, + "loss": 2.4641, + "theoretical_loss": 3.6533588329988604, + "tokens_seen": 988086272 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035386158475426277, + "loss": 2.7381, + "theoretical_loss": 3.6533357226339866, + "tokens_seen": 988151808 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353851554663992, + "loss": 2.7929, + "theoretical_loss": 3.653312614230912, + "tokens_seen": 988217344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035384152457372113, + "loss": 2.8832, + "theoretical_loss": 3.653289507789341, + "tokens_seen": 988282880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035383149448345037, + "loss": 2.6008, + "theoretical_loss": 3.653266403308976, + "tokens_seen": 988348416 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035382146439317955, + "loss": 2.7089, + "theoretical_loss": 3.65324330078952, + "tokens_seen": 988413952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035381143430290873, + "loss": 2.5198, + "theoretical_loss": 3.6532202002306784, + "tokens_seen": 988479488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003538014042126379, + "loss": 2.7445, + "theoretical_loss": 3.653197101632153, + "tokens_seen": 988545024 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537913741223671, + "loss": 2.627, + "theoretical_loss": 3.653174004993648, + "tokens_seen": 988610560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537813440320963, + "loss": 2.7365, + "theoretical_loss": 3.653150910314868, + "tokens_seen": 988676096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537713139418255, + "loss": 2.6624, + "theoretical_loss": 3.653127817595516, + "tokens_seen": 988741632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035376128385155464, + "loss": 2.8601, + "theoretical_loss": 3.6531047268352967, + "tokens_seen": 988807168 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035375125376128387, + "loss": 2.7081, + "theoretical_loss": 3.653081638033913, + "tokens_seen": 988872704 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353741223671013, + "loss": 2.7607, + "theoretical_loss": 3.65305855119107, + "tokens_seen": 988938240 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035373119358074223, + "loss": 2.6208, + "theoretical_loss": 3.6530354663064704, + "tokens_seen": 989003776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537211634904714, + "loss": 2.6153, + "theoretical_loss": 3.65301238337982, + "tokens_seen": 989069312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537111334002006, + "loss": 2.839, + "theoretical_loss": 3.6529893024108215, + "tokens_seen": 989134848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003537011033099298, + "loss": 2.5253, + "theoretical_loss": 3.65296622339918, + "tokens_seen": 989200384 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353691073219659, + "loss": 2.7016, + "theoretical_loss": 3.6529431463446, + "tokens_seen": 989265920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035368104312938814, + "loss": 2.6507, + "theoretical_loss": 3.652920071246786, + "tokens_seen": 989331456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536710130391174, + "loss": 2.4649, + "theoretical_loss": 3.652896998105442, + "tokens_seen": 989396992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536609829488465, + "loss": 2.844, + "theoretical_loss": 3.652873926920272, + "tokens_seen": 989462528 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035365095285857574, + "loss": 2.8394, + "theoretical_loss": 3.652850857690982, + "tokens_seen": 989528064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 553159, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4328830242156982, + "objective/train/theoretical_loss": 3.6528277904172755, + "objective/train/tokens_used": 1010053600, + "theoretical_loss": 3.6528277904172755, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536409227683049, + "loss": 2.5894, + "theoretical_loss": 3.6528277904172755, + "tokens_seen": 989593600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536308926780341, + "loss": 2.3536, + "theoretical_loss": 3.652804725098858, + "tokens_seen": 989659136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536208625877633, + "loss": 2.737, + "theoretical_loss": 3.6527816617354336, + "tokens_seen": 989724672 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035361083249749246, + "loss": 2.4307, + "theoretical_loss": 3.652758600326708, + "tokens_seen": 989790208 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003536008024072217, + "loss": 2.6266, + "theoretical_loss": 3.652735540872385, + "tokens_seen": 989855744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535907723169509, + "loss": 2.5696, + "theoretical_loss": 3.652712483372171, + "tokens_seen": 989921280 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035358074222668006, + "loss": 2.7762, + "theoretical_loss": 3.65268942782577, + "tokens_seen": 989986816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035357071213640924, + "loss": 2.707, + "theoretical_loss": 3.652666374232887, + "tokens_seen": 990052352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535606820461384, + "loss": 2.5712, + "theoretical_loss": 3.652643322593228, + "tokens_seen": 990117888 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535506519558676, + "loss": 2.6914, + "theoretical_loss": 3.652620272906498, + "tokens_seen": 990183424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035354062186559684, + "loss": 2.5723, + "theoretical_loss": 3.6525972251724017, + "tokens_seen": 990248960 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035353059177532597, + "loss": 2.4504, + "theoretical_loss": 3.652574179390645, + "tokens_seen": 990314496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535205616850552, + "loss": 2.5014, + "theoretical_loss": 3.6525511355609335, + "tokens_seen": 990380032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003535105315947844, + "loss": 2.7155, + "theoretical_loss": 3.6525280936829723, + "tokens_seen": 990445568 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035350050150451356, + "loss": 2.6203, + "theoretical_loss": 3.6525050537564674, + "tokens_seen": 990511104 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035349047141424274, + "loss": 2.4302, + "theoretical_loss": 3.652482015781124, + "tokens_seen": 990576640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534804413239719, + "loss": 2.7063, + "theoretical_loss": 3.6524589797566476, + "tokens_seen": 990642176 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534704112337011, + "loss": 2.6519, + "theoretical_loss": 3.6524359456827447, + "tokens_seen": 990707712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035346038114343034, + "loss": 2.5755, + "theoretical_loss": 3.6524129135591203, + "tokens_seen": 990773248 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035345035105315947, + "loss": 2.6261, + "theoretical_loss": 3.652389883385481, + "tokens_seen": 990838784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003534403209628887, + "loss": 2.5731, + "theoretical_loss": 3.6523668551615325, + "tokens_seen": 990904320 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035343029087261783, + "loss": 2.7021, + "theoretical_loss": 3.6523438288869805, + "tokens_seen": 990969856 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035342026078234707, + "loss": 2.7016, + "theoretical_loss": 3.6523208045615316, + "tokens_seen": 991035392 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035341023069207625, + "loss": 2.5318, + "theoretical_loss": 3.652297782184892, + "tokens_seen": 991100928 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035340020060180543, + "loss": 2.6415, + "theoretical_loss": 3.6522747617567672, + "tokens_seen": 991166464 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 554508, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6566503047943115, + "objective/train/theoretical_loss": 3.6522517432768637, + "objective/train/tokens_used": 1011692000, + "theoretical_loss": 3.6522517432768637, + "tokens_seen": 991232000 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533901705115346, + "loss": 2.6658, + "theoretical_loss": 3.6522517432768637, + "tokens_seen": 991232000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035338014042126384, + "loss": 2.6459, + "theoretical_loss": 3.6522287267448883, + "tokens_seen": 991297536 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035337011033099297, + "loss": 2.5836, + "theoretical_loss": 3.652205712160547, + "tokens_seen": 991363072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533600802407222, + "loss": 2.8489, + "theoretical_loss": 3.652182699523546, + "tokens_seen": 991428608 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035335005015045133, + "loss": 2.7108, + "theoretical_loss": 3.6521596888335925, + "tokens_seen": 991494144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035334002006018057, + "loss": 2.5673, + "theoretical_loss": 3.6521366800903925, + "tokens_seen": 991559680 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035332998996990975, + "loss": 2.727, + "theoretical_loss": 3.6521136732936528, + "tokens_seen": 991625216 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035331995987963893, + "loss": 2.7111, + "theoretical_loss": 3.6520906684430807, + "tokens_seen": 991690752 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003533099297893681, + "loss": 2.5756, + "theoretical_loss": 3.652067665538382, + "tokens_seen": 991756288 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532998996990973, + "loss": 2.5974, + "theoretical_loss": 3.652044664579264, + "tokens_seen": 991821824 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532898696088265, + "loss": 2.425, + "theoretical_loss": 3.652021665565434, + "tokens_seen": 991887360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532798395185557, + "loss": 2.6744, + "theoretical_loss": 3.6519986684965984, + "tokens_seen": 991952896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035326980942828484, + "loss": 2.7236, + "theoretical_loss": 3.6519756733724646, + "tokens_seen": 992018432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035325977933801407, + "loss": 2.6378, + "theoretical_loss": 3.651952680192739, + "tokens_seen": 992083968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532497492477432, + "loss": 2.6454, + "theoretical_loss": 3.6519296889571295, + "tokens_seen": 992149504 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035323971915747243, + "loss": 2.6259, + "theoretical_loss": 3.651906699665343, + "tokens_seen": 992215040 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532296890672016, + "loss": 2.5189, + "theoretical_loss": 3.651883712317087, + "tokens_seen": 992280576 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003532196589769308, + "loss": 2.8199, + "theoretical_loss": 3.6518607269120684, + "tokens_seen": 992346112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035320962888666, + "loss": 2.6205, + "theoretical_loss": 3.6518377434499953, + "tokens_seen": 992411648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531995987963892, + "loss": 2.8705, + "theoretical_loss": 3.651814761930575, + "tokens_seen": 992477184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035318956870611834, + "loss": 2.7282, + "theoretical_loss": 3.651791782353514, + "tokens_seen": 992542720 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531795386158476, + "loss": 2.5389, + "theoretical_loss": 3.6517688047185217, + "tokens_seen": 992608256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531695085255767, + "loss": 2.6315, + "theoretical_loss": 3.6517458290253044, + "tokens_seen": 992673792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035315947843530594, + "loss": 2.7756, + "theoretical_loss": 3.6517228552735697, + "tokens_seen": 992739328 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531494483450351, + "loss": 2.6943, + "theoretical_loss": 3.651699883463026, + "tokens_seen": 992804864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 555131, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2285315990448, + "objective/train/theoretical_loss": 3.6516769135933815, + "objective/train/tokens_used": 1013330400, + "theoretical_loss": 3.6516769135933815, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531394182547643, + "loss": 2.5577, + "theoretical_loss": 3.6516769135933815, + "tokens_seen": 992870400 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003531293881644935, + "loss": 2.6386, + "theoretical_loss": 3.6516539456643438, + "tokens_seen": 992935936 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035311935807422266, + "loss": 2.8148, + "theoretical_loss": 3.6516309796756206, + "tokens_seen": 993001472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035310932798395184, + "loss": 2.7412, + "theoretical_loss": 3.65160801562692, + "tokens_seen": 993067008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530992978936811, + "loss": 2.8597, + "theoretical_loss": 3.6515850535179504, + "tokens_seen": 993132544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530892678034102, + "loss": 2.4166, + "theoretical_loss": 3.65156209334842, + "tokens_seen": 993198080 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035307923771313944, + "loss": 2.5507, + "theoretical_loss": 3.651539135118036, + "tokens_seen": 993263616 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035306920762286857, + "loss": 2.7169, + "theoretical_loss": 3.6515161788265083, + "tokens_seen": 993329152 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530591775325978, + "loss": 2.607, + "theoretical_loss": 3.6514932244735436, + "tokens_seen": 993394688 + }, + { + "epoch": 0.3, + "learning_rate": 0.000353049147442327, + "loss": 2.7023, + "theoretical_loss": 3.6514702720588517, + "tokens_seen": 993460224 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035303911735205617, + "loss": 2.3688, + "theoretical_loss": 3.6514473215821406, + "tokens_seen": 993525760 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035302908726178535, + "loss": 2.7075, + "theoretical_loss": 3.6514243730431186, + "tokens_seen": 993591296 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530190571715146, + "loss": 2.4497, + "theoretical_loss": 3.6514014264414953, + "tokens_seen": 993656832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003530090270812437, + "loss": 2.8638, + "theoretical_loss": 3.6513784817769777, + "tokens_seen": 993722368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035299899699097294, + "loss": 2.8829, + "theoretical_loss": 3.6513555390492756, + "tokens_seen": 993787904 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035298896690070207, + "loss": 2.6749, + "theoretical_loss": 3.6513325982580978, + "tokens_seen": 993853440 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529789368104313, + "loss": 2.4084, + "theoretical_loss": 3.651309659403153, + "tokens_seen": 993918976 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529689067201605, + "loss": 2.6559, + "theoretical_loss": 3.6512867224841505, + "tokens_seen": 993984512 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035295887662988967, + "loss": 2.6972, + "theoretical_loss": 3.6512637875007985, + "tokens_seen": 994050048 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035294884653961885, + "loss": 2.8845, + "theoretical_loss": 3.6512408544528063, + "tokens_seen": 994115584 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035293881644934803, + "loss": 2.7391, + "theoretical_loss": 3.6512179233398836, + "tokens_seen": 994181120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529287863590772, + "loss": 2.8006, + "theoretical_loss": 3.651194994161739, + "tokens_seen": 994246656 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035291875626880645, + "loss": 2.6687, + "theoretical_loss": 3.6511720669180816, + "tokens_seen": 994312192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003529087261785356, + "loss": 2.5931, + "theoretical_loss": 3.6511491416086215, + "tokens_seen": 994377728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528986960882648, + "loss": 2.7499, + "theoretical_loss": 3.6511262182330673, + "tokens_seen": 994443264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 556491, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.311375617980957, + "objective/train/theoretical_loss": 3.6511032967911285, + "objective/train/tokens_used": 1014968800, + "theoretical_loss": 3.6511032967911285, + "tokens_seen": 994508800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035288866599799394, + "loss": 2.5432, + "theoretical_loss": 3.6511032967911285, + "tokens_seen": 994508800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035287863590772317, + "loss": 2.6023, + "theoretical_loss": 3.6510803772825153, + "tokens_seen": 994574336 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035286860581745235, + "loss": 2.411, + "theoretical_loss": 3.6510574597069367, + "tokens_seen": 994639872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035285857572718153, + "loss": 2.6745, + "theoretical_loss": 3.6510345440641023, + "tokens_seen": 994705408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035284854563691077, + "loss": 2.8324, + "theoretical_loss": 3.6510116303537217, + "tokens_seen": 994770944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035283851554663995, + "loss": 2.6422, + "theoretical_loss": 3.6509887185755048, + "tokens_seen": 994836480 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035282848545636913, + "loss": 2.4072, + "theoretical_loss": 3.650965808729161, + "tokens_seen": 994902016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528184553660983, + "loss": 2.4776, + "theoretical_loss": 3.6509429008144014, + "tokens_seen": 994967552 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003528084252758275, + "loss": 2.7632, + "theoretical_loss": 3.650919994830935, + "tokens_seen": 995033088 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527983951855567, + "loss": 2.588, + "theoretical_loss": 3.6508970907784715, + "tokens_seen": 995098624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527883650952859, + "loss": 2.5729, + "theoretical_loss": 3.6508741886567213, + "tokens_seen": 995164160 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035277833500501504, + "loss": 2.7314, + "theoretical_loss": 3.650851288465395, + "tokens_seen": 995229696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035276830491474427, + "loss": 2.7634, + "theoretical_loss": 3.650828390204202, + "tokens_seen": 995295232 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527582748244734, + "loss": 2.6234, + "theoretical_loss": 3.650805493872853, + "tokens_seen": 995360768 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035274824473420263, + "loss": 2.8951, + "theoretical_loss": 3.650782599471058, + "tokens_seen": 995426304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527382146439318, + "loss": 2.6232, + "theoretical_loss": 3.650759706998528, + "tokens_seen": 995491840 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352728184553661, + "loss": 2.6352, + "theoretical_loss": 3.6507368164549727, + "tokens_seen": 995557376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527181544633902, + "loss": 2.9256, + "theoretical_loss": 3.650713927840103, + "tokens_seen": 995622912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003527081243731194, + "loss": 2.4447, + "theoretical_loss": 3.650691041153629, + "tokens_seen": 995688448 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035269809428284854, + "loss": 2.6017, + "theoretical_loss": 3.6506681563952625, + "tokens_seen": 995753984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526880641925778, + "loss": 2.6119, + "theoretical_loss": 3.6506452735647126, + "tokens_seen": 995819520 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526780341023069, + "loss": 2.8533, + "theoretical_loss": 3.650622392661691, + "tokens_seen": 995885056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035266800401203614, + "loss": 2.7495, + "theoretical_loss": 3.6505995136859086, + "tokens_seen": 995950592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526579739217653, + "loss": 2.5176, + "theoretical_loss": 3.6505766366370755, + "tokens_seen": 996016128 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526479438314945, + "loss": 2.5834, + "theoretical_loss": 3.6505537615149035, + "tokens_seen": 996081664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 557069, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.978800654411316, + "objective/train/theoretical_loss": 3.650530888319103, + "objective/train/tokens_used": 1016607200, + "theoretical_loss": 3.650530888319103, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526379137412237, + "loss": 2.3804, + "theoretical_loss": 3.650530888319103, + "tokens_seen": 996147200 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035262788365095286, + "loss": 2.7149, + "theoretical_loss": 3.6505080170493853, + "tokens_seen": 996212736 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035261785356068204, + "loss": 2.6825, + "theoretical_loss": 3.6504851477054614, + "tokens_seen": 996278272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003526078234704113, + "loss": 2.9308, + "theoretical_loss": 3.6504622802870426, + "tokens_seen": 996343808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525977933801404, + "loss": 2.7761, + "theoretical_loss": 3.6504394147938406, + "tokens_seen": 996409344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035258776328986964, + "loss": 2.7075, + "theoretical_loss": 3.6504165512255655, + "tokens_seen": 996474880 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035257773319959877, + "loss": 2.6206, + "theoretical_loss": 3.6503936895819296, + "tokens_seen": 996540416 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352567703109328, + "loss": 2.5507, + "theoretical_loss": 3.650370829862644, + "tokens_seen": 996605952 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525576730190572, + "loss": 2.5328, + "theoretical_loss": 3.6503479720674203, + "tokens_seen": 996671488 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035254764292878637, + "loss": 2.7378, + "theoretical_loss": 3.6503251161959707, + "tokens_seen": 996737024 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035253761283851555, + "loss": 2.6422, + "theoretical_loss": 3.6503022622480055, + "tokens_seen": 996802560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525275827482448, + "loss": 2.5801, + "theoretical_loss": 3.650279410223237, + "tokens_seen": 996868096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003525175526579739, + "loss": 2.6606, + "theoretical_loss": 3.6502565601213774, + "tokens_seen": 996933632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035250752256770314, + "loss": 2.4713, + "theoretical_loss": 3.6502337119421377, + "tokens_seen": 996999168 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035249749247743227, + "loss": 2.68, + "theoretical_loss": 3.6502108656852306, + "tokens_seen": 997064704 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524874623871615, + "loss": 2.5103, + "theoretical_loss": 3.6501880213503672, + "tokens_seen": 997130240 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524774322968907, + "loss": 2.6101, + "theoretical_loss": 3.65016517893726, + "tokens_seen": 997195776 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035246740220661987, + "loss": 2.6795, + "theoretical_loss": 3.650142338445621, + "tokens_seen": 997261312 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035245737211634905, + "loss": 2.6608, + "theoretical_loss": 3.6501194998751623, + "tokens_seen": 997326848 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035244734202607823, + "loss": 2.7978, + "theoretical_loss": 3.650096663225596, + "tokens_seen": 997392384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524373119358074, + "loss": 2.7366, + "theoretical_loss": 3.650073828496634, + "tokens_seen": 997457920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035242728184553665, + "loss": 2.9977, + "theoretical_loss": 3.650050995687989, + "tokens_seen": 997523456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003524172517552658, + "loss": 2.9039, + "theoretical_loss": 3.650028164799374, + "tokens_seen": 997588992 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352407221664995, + "loss": 2.6053, + "theoretical_loss": 3.6500053358305005, + "tokens_seen": 997654528 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035239719157472414, + "loss": 2.6113, + "theoretical_loss": 3.649982508781081, + "tokens_seen": 997720064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 558090, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5171451568603516, + "objective/train/theoretical_loss": 3.6499596836508283, + "objective/train/tokens_used": 1018245600, + "theoretical_loss": 3.6499596836508283, + "tokens_seen": 997785600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035238716148445337, + "loss": 2.7921, + "theoretical_loss": 3.6499596836508283, + "tokens_seen": 997785600 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035237713139418255, + "loss": 2.5081, + "theoretical_loss": 3.649936860439455, + "tokens_seen": 997851136 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035236710130391173, + "loss": 2.8577, + "theoretical_loss": 3.6499140391466733, + "tokens_seen": 997916672 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523570712136409, + "loss": 2.6762, + "theoretical_loss": 3.649891219772197, + "tokens_seen": 997982208 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035234704112337015, + "loss": 2.6309, + "theoretical_loss": 3.6498684023157386, + "tokens_seen": 998047744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523370110330993, + "loss": 2.9457, + "theoretical_loss": 3.64984558677701, + "tokens_seen": 998113280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523269809428285, + "loss": 2.7832, + "theoretical_loss": 3.6498227731557247, + "tokens_seen": 998178816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035231695085255764, + "loss": 2.6664, + "theoretical_loss": 3.649799961451596, + "tokens_seen": 998244352 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003523069207622869, + "loss": 2.4468, + "theoretical_loss": 3.6497771516643365, + "tokens_seen": 998309888 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035229689067201606, + "loss": 2.6363, + "theoretical_loss": 3.64975434379366, + "tokens_seen": 998375424 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035228686058174524, + "loss": 2.6484, + "theoretical_loss": 3.649731537839279, + "tokens_seen": 998440960 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522768304914744, + "loss": 2.6946, + "theoretical_loss": 3.6497087338009067, + "tokens_seen": 998506496 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522668004012036, + "loss": 2.8492, + "theoretical_loss": 3.649685931678256, + "tokens_seen": 998572032 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522567703109328, + "loss": 2.3118, + "theoretical_loss": 3.6496631314710415, + "tokens_seen": 998637568 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352246740220662, + "loss": 2.8642, + "theoretical_loss": 3.6496403331789757, + "tokens_seen": 998703104 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035223671013039114, + "loss": 2.4645, + "theoretical_loss": 3.649617536801773, + "tokens_seen": 998768640 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003522266800401204, + "loss": 2.5693, + "theoretical_loss": 3.6495947423391453, + "tokens_seen": 998834176 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035221664994984956, + "loss": 2.7414, + "theoretical_loss": 3.6495719497908077, + "tokens_seen": 998899712 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035220661985957874, + "loss": 2.7699, + "theoretical_loss": 3.6495491591564733, + "tokens_seen": 998965248 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521965897693079, + "loss": 2.698, + "theoretical_loss": 3.6495263704358556, + "tokens_seen": 999030784 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521865596790371, + "loss": 2.4147, + "theoretical_loss": 3.6495035836286687, + "tokens_seen": 999096320 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521765295887663, + "loss": 2.511, + "theoretical_loss": 3.6494807987346265, + "tokens_seen": 999161856 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521664994984955, + "loss": 2.9637, + "theoretical_loss": 3.649458015753442, + "tokens_seen": 999227392 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035215646940822465, + "loss": 2.5515, + "theoretical_loss": 3.649435234684831, + "tokens_seen": 999292928 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521464393179539, + "loss": 2.7807, + "theoretical_loss": 3.6494124555285063, + "tokens_seen": 999358464 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 559222, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.206585645675659, + "objective/train/theoretical_loss": 3.649389678284182, + "objective/train/tokens_used": 1019884000, + "theoretical_loss": 3.649389678284182, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352136409227683, + "loss": 2.775, + "theoretical_loss": 3.649389678284182, + "tokens_seen": 999424000 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035212637913741224, + "loss": 2.7452, + "theoretical_loss": 3.649366902951572, + "tokens_seen": 999489536 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521163490471414, + "loss": 2.6143, + "theoretical_loss": 3.649344129530391, + "tokens_seen": 999555072 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003521063189568706, + "loss": 2.8037, + "theoretical_loss": 3.6493213580203534, + "tokens_seen": 999620608 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035209628886659984, + "loss": 2.6248, + "theoretical_loss": 3.6492985884211735, + "tokens_seen": 999686144 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035208625877632897, + "loss": 3.0173, + "theoretical_loss": 3.6492758207325657, + "tokens_seen": 999751680 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520762286860582, + "loss": 2.7024, + "theoretical_loss": 3.6492530549542437, + "tokens_seen": 999817216 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520661985957874, + "loss": 2.943, + "theoretical_loss": 3.6492302910859227, + "tokens_seen": 999882752 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035205616850551657, + "loss": 3.0572, + "theoretical_loss": 3.6492075291273176, + "tokens_seen": 999948288 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035204613841524575, + "loss": 2.9415, + "theoretical_loss": 3.6491847690781425, + "tokens_seen": 1000013824 + }, + { + "epoch": 0.3, + "learning_rate": 0.000352036108324975, + "loss": 2.6723, + "theoretical_loss": 3.649162010938112, + "tokens_seen": 1000079360 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003520260782347041, + "loss": 2.6927, + "theoretical_loss": 3.6491392547069412, + "tokens_seen": 1000144896 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035201604814443334, + "loss": 2.7726, + "theoretical_loss": 3.649116500384345, + "tokens_seen": 1000210432 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035200601805416247, + "loss": 2.7641, + "theoretical_loss": 3.6490937479700376, + "tokens_seen": 1000275968 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519959879638917, + "loss": 2.7668, + "theoretical_loss": 3.649070997463735, + "tokens_seen": 1000341504 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519859578736209, + "loss": 2.6113, + "theoretical_loss": 3.649048248865152, + "tokens_seen": 1000407040 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035197592778335007, + "loss": 2.6915, + "theoretical_loss": 3.649025502174003, + "tokens_seen": 1000472576 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035196589769307925, + "loss": 2.7293, + "theoretical_loss": 3.649002757390003, + "tokens_seen": 1000538112 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035195586760280843, + "loss": 2.4951, + "theoretical_loss": 3.648980014512868, + "tokens_seen": 1000603648 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519458375125376, + "loss": 2.6436, + "theoretical_loss": 3.648957273542313, + "tokens_seen": 1000669184 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035193580742226685, + "loss": 2.6603, + "theoretical_loss": 3.6489345344780535, + "tokens_seen": 1000734720 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351925777331996, + "loss": 2.6807, + "theoretical_loss": 3.648911797319804, + "tokens_seen": 1000800256 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003519157472417252, + "loss": 2.4684, + "theoretical_loss": 3.648889062067281, + "tokens_seen": 1000865792 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035190571715145434, + "loss": 2.5266, + "theoretical_loss": 3.6488663287202, + "tokens_seen": 1000931328 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035189568706118357, + "loss": 2.7564, + "theoretical_loss": 3.6488435972782756, + "tokens_seen": 1000996864 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 559970, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.601503372192383, + "objective/train/theoretical_loss": 3.6488208677412244, + "objective/train/tokens_used": 1021522400, + "theoretical_loss": 3.6488208677412244, + "tokens_seen": 1001062400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035188565697091275, + "loss": 2.6304, + "theoretical_loss": 3.6488208677412244, + "tokens_seen": 1001062400 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035187562688064193, + "loss": 2.9246, + "theoretical_loss": 3.6487981401087612, + "tokens_seen": 1001127936 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518655967903711, + "loss": 2.6429, + "theoretical_loss": 3.6487754143806024, + "tokens_seen": 1001193472 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035185556670010035, + "loss": 2.6234, + "theoretical_loss": 3.6487526905564636, + "tokens_seen": 1001259008 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518455366098295, + "loss": 2.7218, + "theoretical_loss": 3.64872996863606, + "tokens_seen": 1001324544 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518355065195587, + "loss": 2.9734, + "theoretical_loss": 3.648707248619109, + "tokens_seen": 1001390080 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035182547642928784, + "loss": 2.7579, + "theoretical_loss": 3.6486845305053253, + "tokens_seen": 1001455616 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003518154463390171, + "loss": 2.746, + "theoretical_loss": 3.6486618142944263, + "tokens_seen": 1001521152 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035180541624874626, + "loss": 2.7835, + "theoretical_loss": 3.6486390999861262, + "tokens_seen": 1001586688 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035179538615847544, + "loss": 2.7529, + "theoretical_loss": 3.6486163875801427, + "tokens_seen": 1001652224 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517853560682046, + "loss": 2.6583, + "theoretical_loss": 3.6485936770761915, + "tokens_seen": 1001717760 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517753259779338, + "loss": 2.7264, + "theoretical_loss": 3.6485709684739884, + "tokens_seen": 1001783296 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351765295887663, + "loss": 2.4917, + "theoretical_loss": 3.648548261773251, + "tokens_seen": 1001848832 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517552657973922, + "loss": 2.6531, + "theoretical_loss": 3.648525556973695, + "tokens_seen": 1001914368 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035174523570712134, + "loss": 2.809, + "theoretical_loss": 3.648502854075037, + "tokens_seen": 1001979904 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517352056168506, + "loss": 2.6379, + "theoretical_loss": 3.648480153076993, + "tokens_seen": 1002045440 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035172517552657976, + "loss": 2.6711, + "theoretical_loss": 3.64845745397928, + "tokens_seen": 1002110976 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035171514543630894, + "loss": 2.7953, + "theoretical_loss": 3.6484347567816147, + "tokens_seen": 1002176512 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003517051153460381, + "loss": 2.6388, + "theoretical_loss": 3.648412061483714, + "tokens_seen": 1002242048 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516950852557673, + "loss": 2.6233, + "theoretical_loss": 3.6483893680852946, + "tokens_seen": 1002307584 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516850551654965, + "loss": 2.7684, + "theoretical_loss": 3.6483666765860727, + "tokens_seen": 1002373120 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516750250752257, + "loss": 2.4296, + "theoretical_loss": 3.648343986985766, + "tokens_seen": 1002438656 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035166499498495485, + "loss": 2.7591, + "theoretical_loss": 3.6483212992840914, + "tokens_seen": 1002504192 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516549648946841, + "loss": 2.5995, + "theoretical_loss": 3.6482986134807653, + "tokens_seen": 1002569728 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516449348044132, + "loss": 2.9894, + "theoretical_loss": 3.6482759295755054, + "tokens_seen": 1002635264 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 560865, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5041370391845703, + "objective/train/theoretical_loss": 3.6482532475680287, + "objective/train/tokens_used": 1023160800, + "theoretical_loss": 3.6482532475680287, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035163490471414244, + "loss": 2.7673, + "theoretical_loss": 3.6482532475680287, + "tokens_seen": 1002700800 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516248746238716, + "loss": 2.7331, + "theoretical_loss": 3.6482305674580524, + "tokens_seen": 1002766336 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003516148445336008, + "loss": 2.5743, + "theoretical_loss": 3.6482078892452936, + "tokens_seen": 1002831872 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035160481444333, + "loss": 2.6882, + "theoretical_loss": 3.64818521292947, + "tokens_seen": 1002897408 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035159478435305917, + "loss": 2.816, + "theoretical_loss": 3.648162538510298, + "tokens_seen": 1002962944 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035158475426278835, + "loss": 2.6259, + "theoretical_loss": 3.6481398659874964, + "tokens_seen": 1003028480 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515747241725176, + "loss": 2.8063, + "theoretical_loss": 3.648117195360782, + "tokens_seen": 1003094016 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515646940822467, + "loss": 2.7044, + "theoretical_loss": 3.6480945266298725, + "tokens_seen": 1003159552 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035155466399197595, + "loss": 2.5904, + "theoretical_loss": 3.6480718597944852, + "tokens_seen": 1003225088 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035154463390170513, + "loss": 2.7078, + "theoretical_loss": 3.648049194854338, + "tokens_seen": 1003290624 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515346038114343, + "loss": 2.7708, + "theoretical_loss": 3.6480265318091485, + "tokens_seen": 1003356160 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003515245737211635, + "loss": 2.6009, + "theoretical_loss": 3.6480038706586355, + "tokens_seen": 1003421696 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035151454363089267, + "loss": 2.866, + "theoretical_loss": 3.6479812114025156, + "tokens_seen": 1003487232 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035150451354062185, + "loss": 2.6386, + "theoretical_loss": 3.6479585540405073, + "tokens_seen": 1003552768 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514944834503511, + "loss": 2.6617, + "theoretical_loss": 3.6479358985723285, + "tokens_seen": 1003618304 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514844533600802, + "loss": 2.7367, + "theoretical_loss": 3.647913244997697, + "tokens_seen": 1003683840 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035147442326980945, + "loss": 2.7202, + "theoretical_loss": 3.6478905933163315, + "tokens_seen": 1003749376 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514643931795386, + "loss": 2.9854, + "theoretical_loss": 3.6478679435279497, + "tokens_seen": 1003814912 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514543630892678, + "loss": 2.692, + "theoretical_loss": 3.6478452956322704, + "tokens_seen": 1003880448 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351444332998997, + "loss": 2.6885, + "theoretical_loss": 3.647822649629011, + "tokens_seen": 1003945984 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514343029087262, + "loss": 2.9112, + "theoretical_loss": 3.6478000055178907, + "tokens_seen": 1004011520 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035142427281845536, + "loss": 2.8827, + "theoretical_loss": 3.6477773632986272, + "tokens_seen": 1004077056 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035141424272818454, + "loss": 2.5035, + "theoretical_loss": 3.6477547229709395, + "tokens_seen": 1004142592 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003514042126379137, + "loss": 2.6845, + "theoretical_loss": 3.647732084534546, + "tokens_seen": 1004208128 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035139418254764295, + "loss": 2.8052, + "theoretical_loss": 3.647709447989165, + "tokens_seen": 1004273664 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 561903, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.305278778076172, + "objective/train/theoretical_loss": 3.6476868133345155, + "objective/train/tokens_used": 1024799200, + "theoretical_loss": 3.6476868133345155, + "tokens_seen": 1004339200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513841524573721, + "loss": 2.7461, + "theoretical_loss": 3.6476868133345155, + "tokens_seen": 1004339200 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513741223671013, + "loss": 2.505, + "theoretical_loss": 3.6476641805703163, + "tokens_seen": 1004404736 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513640922768305, + "loss": 2.6162, + "theoretical_loss": 3.647641549696286, + "tokens_seen": 1004470272 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513540621865597, + "loss": 2.715, + "theoretical_loss": 3.6476189207121434, + "tokens_seen": 1004535808 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513440320962889, + "loss": 2.6644, + "theoretical_loss": 3.647596293617607, + "tokens_seen": 1004601344 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035133400200601804, + "loss": 2.3282, + "theoretical_loss": 3.647573668412397, + "tokens_seen": 1004666880 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003513239719157473, + "loss": 2.8704, + "theoretical_loss": 3.6475510450962307, + "tokens_seen": 1004732416 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035131394182547646, + "loss": 2.9088, + "theoretical_loss": 3.6475284236688283, + "tokens_seen": 1004797952 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035130391173520564, + "loss": 2.5969, + "theoretical_loss": 3.647505804129909, + "tokens_seen": 1004863488 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512938816449348, + "loss": 2.7106, + "theoretical_loss": 3.647483186479192, + "tokens_seen": 1004929024 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351283851554664, + "loss": 2.6929, + "theoretical_loss": 3.6474605707163956, + "tokens_seen": 1004994560 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512738214643932, + "loss": 2.7415, + "theoretical_loss": 3.64743795684124, + "tokens_seen": 1005060096 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512637913741224, + "loss": 2.8185, + "theoretical_loss": 3.6474153448534445, + "tokens_seen": 1005125632 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035125376128385154, + "loss": 2.6448, + "theoretical_loss": 3.6473927347527284, + "tokens_seen": 1005191168 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512437311935808, + "loss": 2.8856, + "theoretical_loss": 3.647370126538811, + "tokens_seen": 1005256704 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035123370110330996, + "loss": 2.484, + "theoretical_loss": 3.647347520211412, + "tokens_seen": 1005322240 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035122367101303914, + "loss": 2.9744, + "theoretical_loss": 3.647324915770251, + "tokens_seen": 1005387776 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512136409227683, + "loss": 2.7492, + "theoretical_loss": 3.6473023132150484, + "tokens_seen": 1005453312 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003512036108324975, + "loss": 2.5703, + "theoretical_loss": 3.6472797125455227, + "tokens_seen": 1005518848 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511935807422267, + "loss": 2.8978, + "theoretical_loss": 3.6472571137613943, + "tokens_seen": 1005584384 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511835506519559, + "loss": 2.9041, + "theoretical_loss": 3.647234516862383, + "tokens_seen": 1005649920 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035117352056168505, + "loss": 2.6811, + "theoretical_loss": 3.647211921848209, + "tokens_seen": 1005715456 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511634904714143, + "loss": 2.6016, + "theoretical_loss": 3.647189328718592, + "tokens_seen": 1005780992 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511534603811434, + "loss": 2.8905, + "theoretical_loss": 3.6471667374732517, + "tokens_seen": 1005846528 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035114343029087264, + "loss": 2.6996, + "theoretical_loss": 3.6471441481119093, + "tokens_seen": 1005912064 + }, + { + "epoch": 0.3, + "objective/train/docs_used": 562546, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.896876335144043, + "objective/train/theoretical_loss": 3.6471215606342833, + "objective/train/tokens_used": 1026437600, + "theoretical_loss": 3.6471215606342833, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511334002006018, + "loss": 2.8747, + "theoretical_loss": 3.6471215606342833, + "tokens_seen": 1005977600 + }, + { + "epoch": 0.3, + "learning_rate": 0.000351123370110331, + "loss": 2.5018, + "theoretical_loss": 3.6470989750400955, + "tokens_seen": 1006043136 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003511133400200602, + "loss": 2.5671, + "theoretical_loss": 3.6470763913290654, + "tokens_seen": 1006108672 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035110330992978937, + "loss": 2.6394, + "theoretical_loss": 3.647053809500913, + "tokens_seen": 1006174208 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035109327983951855, + "loss": 2.7279, + "theoretical_loss": 3.6470312295553597, + "tokens_seen": 1006239744 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510832497492478, + "loss": 2.6031, + "theoretical_loss": 3.6470086514921247, + "tokens_seen": 1006305280 + }, + { + "epoch": 0.3, + "learning_rate": 0.0003510732196589769, + "loss": 2.6531, + "theoretical_loss": 3.6469860753109296, + "tokens_seen": 1006370816 + }, + { + "epoch": 0.3, + "learning_rate": 0.00035106318956870615, + "loss": 2.7165, + "theoretical_loss": 3.6469635010114945, + "tokens_seen": 1006436352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035105315947843533, + "loss": 2.9426, + "theoretical_loss": 3.64694092859354, + "tokens_seen": 1006501888 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003510431293881645, + "loss": 2.9879, + "theoretical_loss": 3.646918358056787, + "tokens_seen": 1006567424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003510330992978937, + "loss": 2.6251, + "theoretical_loss": 3.6468957894009564, + "tokens_seen": 1006632960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035102306920762287, + "loss": 2.629, + "theoretical_loss": 3.6468732226257687, + "tokens_seen": 1006698496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035101303911735205, + "loss": 2.7709, + "theoretical_loss": 3.6468506577309445, + "tokens_seen": 1006764032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003510030090270813, + "loss": 2.7407, + "theoretical_loss": 3.6468280947162053, + "tokens_seen": 1006829568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509929789368104, + "loss": 2.6753, + "theoretical_loss": 3.6468055335812717, + "tokens_seen": 1006895104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035098294884653965, + "loss": 2.6529, + "theoretical_loss": 3.6467829743258653, + "tokens_seen": 1006960640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509729187562688, + "loss": 2.5762, + "theoretical_loss": 3.6467604169497063, + "tokens_seen": 1007026176 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350962888665998, + "loss": 2.9005, + "theoretical_loss": 3.6467378614525163, + "tokens_seen": 1007091712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509528585757272, + "loss": 2.7911, + "theoretical_loss": 3.6467153078340173, + "tokens_seen": 1007157248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509428284854564, + "loss": 2.5908, + "theoretical_loss": 3.6466927560939295, + "tokens_seen": 1007222784 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035093279839518556, + "loss": 2.6977, + "theoretical_loss": 3.6466702062319745, + "tokens_seen": 1007288320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035092276830491474, + "loss": 2.7994, + "theoretical_loss": 3.646647658247874, + "tokens_seen": 1007353856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003509127382146439, + "loss": 2.5597, + "theoretical_loss": 3.6466251121413493, + "tokens_seen": 1007419392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035090270812437315, + "loss": 2.5898, + "theoretical_loss": 3.6466025679121223, + "tokens_seen": 1007484928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508926780341023, + "loss": 2.6098, + "theoretical_loss": 3.646580025559914, + "tokens_seen": 1007550464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 563033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2577614784240723, + "objective/train/theoretical_loss": 3.646557485084446, + "objective/train/tokens_used": 1028076000, + "theoretical_loss": 3.646557485084446, + "tokens_seen": 1007616000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508826479438315, + "loss": 2.7075, + "theoretical_loss": 3.646557485084446, + "tokens_seen": 1007616000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508726178535607, + "loss": 2.5445, + "theoretical_loss": 3.6465349464854406, + "tokens_seen": 1007681536 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508625877632899, + "loss": 2.6022, + "theoretical_loss": 3.646512409762619, + "tokens_seen": 1007747072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035085255767301906, + "loss": 2.8948, + "theoretical_loss": 3.6464898749157033, + "tokens_seen": 1007812608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035084252758274824, + "loss": 2.8789, + "theoretical_loss": 3.646467341944416, + "tokens_seen": 1007878144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508324974924774, + "loss": 2.4769, + "theoretical_loss": 3.6464448108484775, + "tokens_seen": 1007943680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035082246740220666, + "loss": 2.6822, + "theoretical_loss": 3.6464222816276113, + "tokens_seen": 1008009216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003508124373119358, + "loss": 2.6443, + "theoretical_loss": 3.6463997542815383, + "tokens_seen": 1008074752 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350802407221665, + "loss": 2.7836, + "theoretical_loss": 3.646377228809982, + "tokens_seen": 1008140288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035079237713139415, + "loss": 2.4362, + "theoretical_loss": 3.646354705212663, + "tokens_seen": 1008205824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507823470411234, + "loss": 2.6115, + "theoretical_loss": 3.6463321834893048, + "tokens_seen": 1008271360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035077231695085256, + "loss": 2.6302, + "theoretical_loss": 3.646309663639629, + "tokens_seen": 1008336896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035076228686058174, + "loss": 2.5671, + "theoretical_loss": 3.646287145663358, + "tokens_seen": 1008402432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507522567703109, + "loss": 2.6446, + "theoretical_loss": 3.6462646295602146, + "tokens_seen": 1008467968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035074222668004016, + "loss": 2.787, + "theoretical_loss": 3.6462421153299207, + "tokens_seen": 1008533504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507321965897693, + "loss": 2.3819, + "theoretical_loss": 3.6462196029721996, + "tokens_seen": 1008599040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507221664994985, + "loss": 2.6316, + "theoretical_loss": 3.6461970924867737, + "tokens_seen": 1008664576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035071213640922765, + "loss": 2.8287, + "theoretical_loss": 3.646174583873365, + "tokens_seen": 1008730112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003507021063189569, + "loss": 2.8155, + "theoretical_loss": 3.6461520771316964, + "tokens_seen": 1008795648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035069207622868607, + "loss": 2.7533, + "theoretical_loss": 3.646129572261491, + "tokens_seen": 1008861184 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035068204613841525, + "loss": 2.8465, + "theoretical_loss": 3.646107069262472, + "tokens_seen": 1008926720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035067201604814443, + "loss": 2.5893, + "theoretical_loss": 3.6460845681343614, + "tokens_seen": 1008992256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506619859578736, + "loss": 2.809, + "theoretical_loss": 3.646062068876883, + "tokens_seen": 1009057792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506519558676028, + "loss": 2.8392, + "theoretical_loss": 3.646039571489759, + "tokens_seen": 1009123328 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350641925777332, + "loss": 2.8639, + "theoretical_loss": 3.646017075972713, + "tokens_seen": 1009188864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 564080, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.776188373565674, + "objective/train/theoretical_loss": 3.645994582325468, + "objective/train/tokens_used": 1029714400, + "theoretical_loss": 3.645994582325468, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035063189568706115, + "loss": 2.9152, + "theoretical_loss": 3.645994582325468, + "tokens_seen": 1009254400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506218655967904, + "loss": 2.8531, + "theoretical_loss": 3.6459720905477466, + "tokens_seen": 1009319936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003506118355065195, + "loss": 2.7551, + "theoretical_loss": 3.6459496006392733, + "tokens_seen": 1009385472 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035060180541624875, + "loss": 2.5909, + "theoretical_loss": 3.6459271125997708, + "tokens_seen": 1009451008 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350591775325978, + "loss": 2.5893, + "theoretical_loss": 3.645904626428962, + "tokens_seen": 1009516544 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505817452357071, + "loss": 2.863, + "theoretical_loss": 3.6458821421265704, + "tokens_seen": 1009582080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035057171514543635, + "loss": 2.9149, + "theoretical_loss": 3.64585965969232, + "tokens_seen": 1009647616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035056168505516553, + "loss": 2.4815, + "theoretical_loss": 3.6458371791259343, + "tokens_seen": 1009713152 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505516549648947, + "loss": 2.8522, + "theoretical_loss": 3.6458147004271364, + "tokens_seen": 1009778688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505416248746239, + "loss": 2.4721, + "theoretical_loss": 3.6457922235956506, + "tokens_seen": 1009844224 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035053159478435307, + "loss": 2.6782, + "theoretical_loss": 3.6457697486312, + "tokens_seen": 1009909760 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035052156469408225, + "loss": 2.978, + "theoretical_loss": 3.645747275533509, + "tokens_seen": 1009975296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505115346038115, + "loss": 2.7994, + "theoretical_loss": 3.6457248043023007, + "tokens_seen": 1010040832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003505015045135406, + "loss": 2.6268, + "theoretical_loss": 3.6457023349372992, + "tokens_seen": 1010106368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035049147442326985, + "loss": 2.7156, + "theoretical_loss": 3.645679867438229, + "tokens_seen": 1010171904 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350481444332999, + "loss": 2.7835, + "theoretical_loss": 3.6456574018048133, + "tokens_seen": 1010237440 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504714142427282, + "loss": 2.8286, + "theoretical_loss": 3.6456349380367774, + "tokens_seen": 1010302976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504613841524574, + "loss": 2.8143, + "theoretical_loss": 3.6456124761338433, + "tokens_seen": 1010368512 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504513540621866, + "loss": 2.613, + "theoretical_loss": 3.6455900160957375, + "tokens_seen": 1010434048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035044132397191576, + "loss": 2.484, + "theoretical_loss": 3.645567557922183, + "tokens_seen": 1010499584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035043129388164494, + "loss": 2.7924, + "theoretical_loss": 3.645545101612904, + "tokens_seen": 1010565120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504212637913741, + "loss": 2.7572, + "theoretical_loss": 3.645522647167626, + "tokens_seen": 1010630656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035041123370110335, + "loss": 2.6412, + "theoretical_loss": 3.645500194586072, + "tokens_seen": 1010696192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003504012036108325, + "loss": 2.8669, + "theoretical_loss": 3.645477743867967, + "tokens_seen": 1010761728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503911735205617, + "loss": 2.8419, + "theoretical_loss": 3.645455295013036, + "tokens_seen": 1010827264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 564461, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.960507869720459, + "objective/train/theoretical_loss": 3.6454328480210028, + "objective/train/tokens_used": 1031352800, + "theoretical_loss": 3.6454328480210028, + "tokens_seen": 1010892800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503811434302909, + "loss": 2.9234, + "theoretical_loss": 3.6454328480210028, + "tokens_seen": 1010892800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503711133400201, + "loss": 2.9007, + "theoretical_loss": 3.6454104028915926, + "tokens_seen": 1010958336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035036108324974926, + "loss": 2.8377, + "theoretical_loss": 3.64538795962453, + "tokens_seen": 1011023872 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035035105315947844, + "loss": 2.7468, + "theoretical_loss": 3.64536551821954, + "tokens_seen": 1011089408 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503410230692076, + "loss": 2.8693, + "theoretical_loss": 3.6453430786763468, + "tokens_seen": 1011154944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035033099297893686, + "loss": 2.4695, + "theoretical_loss": 3.645320640994676, + "tokens_seen": 1011220480 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350320962888666, + "loss": 2.6836, + "theoretical_loss": 3.645298205174252, + "tokens_seen": 1011286016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003503109327983952, + "loss": 2.7281, + "theoretical_loss": 3.6452757712148, + "tokens_seen": 1011351552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035030090270812435, + "loss": 2.7976, + "theoretical_loss": 3.645253339116045, + "tokens_seen": 1011417088 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502908726178536, + "loss": 2.6928, + "theoretical_loss": 3.6452309088777124, + "tokens_seen": 1011482624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035028084252758276, + "loss": 2.4886, + "theoretical_loss": 3.645208480499527, + "tokens_seen": 1011548160 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035027081243731194, + "loss": 2.6801, + "theoretical_loss": 3.6451860539812144, + "tokens_seen": 1011613696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502607823470411, + "loss": 2.7794, + "theoretical_loss": 3.6451636293225, + "tokens_seen": 1011679232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035025075225677036, + "loss": 2.5784, + "theoretical_loss": 3.6451412065231086, + "tokens_seen": 1011744768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502407221664995, + "loss": 2.7135, + "theoretical_loss": 3.645118785582765, + "tokens_seen": 1011810304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502306920762287, + "loss": 2.6828, + "theoretical_loss": 3.6450963665011966, + "tokens_seen": 1011875840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035022066198595785, + "loss": 2.8697, + "theoretical_loss": 3.6450739492781277, + "tokens_seen": 1011941376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003502106318956871, + "loss": 2.6464, + "theoretical_loss": 3.645051533913284, + "tokens_seen": 1012006912 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035020060180541627, + "loss": 2.7643, + "theoretical_loss": 3.6450291204063907, + "tokens_seen": 1012072448 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035019057171514545, + "loss": 2.6983, + "theoretical_loss": 3.645006708757175, + "tokens_seen": 1012137984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035018054162487463, + "loss": 2.6446, + "theoretical_loss": 3.644984298965361, + "tokens_seen": 1012203520 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501705115346038, + "loss": 2.5605, + "theoretical_loss": 3.6449618910306754, + "tokens_seen": 1012269056 + }, + { + "epoch": 0.31, + "learning_rate": 0.000350160481444333, + "loss": 2.5893, + "theoretical_loss": 3.6449394849528436, + "tokens_seen": 1012334592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501504513540622, + "loss": 2.6389, + "theoretical_loss": 3.644917080731592, + "tokens_seen": 1012400128 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035014042126379135, + "loss": 2.7151, + "theoretical_loss": 3.6448946783666463, + "tokens_seen": 1012465664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 565625, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7371182441711426, + "objective/train/theoretical_loss": 3.6448722778577327, + "objective/train/tokens_used": 1032991200, + "theoretical_loss": 3.6448722778577327, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501303911735206, + "loss": 2.836, + "theoretical_loss": 3.6448722778577327, + "tokens_seen": 1012531200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003501203610832497, + "loss": 2.6583, + "theoretical_loss": 3.6448498792045774, + "tokens_seen": 1012596736 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035011033099297895, + "loss": 2.5067, + "theoretical_loss": 3.6448274824069067, + "tokens_seen": 1012662272 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035010030090270813, + "loss": 2.6818, + "theoretical_loss": 3.644805087464446, + "tokens_seen": 1012727808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500902708124373, + "loss": 2.3187, + "theoretical_loss": 3.644782694376923, + "tokens_seen": 1012793344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500802407221665, + "loss": 2.6766, + "theoretical_loss": 3.6447603031440625, + "tokens_seen": 1012858880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035007021063189573, + "loss": 2.4468, + "theoretical_loss": 3.6447379137655913, + "tokens_seen": 1012924416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035006018054162486, + "loss": 2.6092, + "theoretical_loss": 3.644715526241237, + "tokens_seen": 1012989952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500501504513541, + "loss": 2.5698, + "theoretical_loss": 3.6446931405707246, + "tokens_seen": 1013055488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500401203610832, + "loss": 2.7784, + "theoretical_loss": 3.644670756753782, + "tokens_seen": 1013121024 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035003009027081245, + "loss": 2.5981, + "theoretical_loss": 3.6446483747901346, + "tokens_seen": 1013186560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035002006018054163, + "loss": 2.6603, + "theoretical_loss": 3.64462599467951, + "tokens_seen": 1013252096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003500100300902708, + "loss": 2.9651, + "theoretical_loss": 3.644603616421634, + "tokens_seen": 1013317632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00035, + "loss": 2.5475, + "theoretical_loss": 3.644581240016235, + "tokens_seen": 1013383168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499899699097292, + "loss": 2.9199, + "theoretical_loss": 3.644558865463038, + "tokens_seen": 1013448704 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034997993981945836, + "loss": 2.6318, + "theoretical_loss": 3.6445364927617714, + "tokens_seen": 1013514240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499699097291876, + "loss": 2.6628, + "theoretical_loss": 3.6445141219121617, + "tokens_seen": 1013579776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499598796389167, + "loss": 2.595, + "theoretical_loss": 3.6444917529139356, + "tokens_seen": 1013645312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034994984954864596, + "loss": 2.8204, + "theoretical_loss": 3.6444693857668207, + "tokens_seen": 1013710848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499398194583751, + "loss": 2.8346, + "theoretical_loss": 3.6444470204705435, + "tokens_seen": 1013776384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499297893681043, + "loss": 2.7842, + "theoretical_loss": 3.6444246570248318, + "tokens_seen": 1013841920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499197592778335, + "loss": 2.6016, + "theoretical_loss": 3.644402295429413, + "tokens_seen": 1013907456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003499097291875627, + "loss": 2.818, + "theoretical_loss": 3.6443799356840136, + "tokens_seen": 1013972992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034989969909729186, + "loss": 2.5587, + "theoretical_loss": 3.6443575777883614, + "tokens_seen": 1014038528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498896690070211, + "loss": 2.6353, + "theoretical_loss": 3.6443352217421845, + "tokens_seen": 1014104064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 566086, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7045507431030273, + "objective/train/theoretical_loss": 3.6443128675452092, + "objective/train/tokens_used": 1034629600, + "theoretical_loss": 3.6443128675452092, + "tokens_seen": 1014169600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498796389167502, + "loss": 2.6942, + "theoretical_loss": 3.6443128675452092, + "tokens_seen": 1014169600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034986960882647946, + "loss": 2.8548, + "theoretical_loss": 3.6442905151971647, + "tokens_seen": 1014235136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498595787362086, + "loss": 2.7197, + "theoretical_loss": 3.6442681646977766, + "tokens_seen": 1014300672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498495486459378, + "loss": 2.6459, + "theoretical_loss": 3.6442458160467743, + "tokens_seen": 1014366208 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034983951855566706, + "loss": 2.5951, + "theoretical_loss": 3.644223469243885, + "tokens_seen": 1014431744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498294884653962, + "loss": 2.7552, + "theoretical_loss": 3.644201124288836, + "tokens_seen": 1014497280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003498194583751254, + "loss": 2.7001, + "theoretical_loss": 3.6441787811813553, + "tokens_seen": 1014562816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034980942828485455, + "loss": 2.7129, + "theoretical_loss": 3.6441564399211708, + "tokens_seen": 1014628352 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497993981945838, + "loss": 2.6051, + "theoretical_loss": 3.644134100508011, + "tokens_seen": 1014693888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034978936810431296, + "loss": 2.6101, + "theoretical_loss": 3.6441117629416038, + "tokens_seen": 1014759424 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034977933801404214, + "loss": 2.6598, + "theoretical_loss": 3.644089427221677, + "tokens_seen": 1014824960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497693079237713, + "loss": 2.4322, + "theoretical_loss": 3.644067093347958, + "tokens_seen": 1014890496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034975927783350056, + "loss": 2.7851, + "theoretical_loss": 3.6440447613201767, + "tokens_seen": 1014956032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497492477432297, + "loss": 2.7539, + "theoretical_loss": 3.64402243113806, + "tokens_seen": 1015021568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497392176529589, + "loss": 2.5749, + "theoretical_loss": 3.6440001028013365, + "tokens_seen": 1015087104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034972918756268805, + "loss": 2.8158, + "theoretical_loss": 3.643977776309735, + "tokens_seen": 1015152640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003497191574724173, + "loss": 2.753, + "theoretical_loss": 3.6439554516629835, + "tokens_seen": 1015218176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034970912738214647, + "loss": 2.7789, + "theoretical_loss": 3.6439331288608106, + "tokens_seen": 1015283712 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034969909729187565, + "loss": 2.6908, + "theoretical_loss": 3.6439108079029445, + "tokens_seen": 1015349248 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034968906720160483, + "loss": 2.742, + "theoretical_loss": 3.643888488789115, + "tokens_seen": 1015414784 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349679037111334, + "loss": 2.7681, + "theoretical_loss": 3.6438661715190492, + "tokens_seen": 1015480320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496690070210632, + "loss": 2.8472, + "theoretical_loss": 3.643843856092477, + "tokens_seen": 1015545856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496589769307924, + "loss": 2.5085, + "theoretical_loss": 3.6438215425091265, + "tokens_seen": 1015611392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034964894684052155, + "loss": 2.9066, + "theoretical_loss": 3.6437992307687264, + "tokens_seen": 1015676928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496389167502508, + "loss": 2.673, + "theoretical_loss": 3.6437769208710065, + "tokens_seen": 1015742464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 567051, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.23244047164917, + "objective/train/theoretical_loss": 3.6437546128156946, + "objective/train/tokens_used": 1036268000, + "theoretical_loss": 3.6437546128156946, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003496288866599799, + "loss": 2.4863, + "theoretical_loss": 3.6437546128156946, + "tokens_seen": 1015808000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034961885656970915, + "loss": 2.7115, + "theoretical_loss": 3.6437323066025207, + "tokens_seen": 1015873536 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034960882647943833, + "loss": 2.6778, + "theoretical_loss": 3.643710002231213, + "tokens_seen": 1015939072 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495987963891675, + "loss": 2.7031, + "theoretical_loss": 3.643687699701501, + "tokens_seen": 1016004608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495887662988967, + "loss": 2.8153, + "theoretical_loss": 3.6436653990131145, + "tokens_seen": 1016070144 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034957873620862593, + "loss": 2.6424, + "theoretical_loss": 3.6436431001657814, + "tokens_seen": 1016135680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034956870611835506, + "loss": 2.6089, + "theoretical_loss": 3.643620803159232, + "tokens_seen": 1016201216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495586760280843, + "loss": 2.6988, + "theoretical_loss": 3.643598507993195, + "tokens_seen": 1016266752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495486459378134, + "loss": 2.7106, + "theoretical_loss": 3.6435762146674007, + "tokens_seen": 1016332288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034953861584754265, + "loss": 2.4762, + "theoretical_loss": 3.6435539231815777, + "tokens_seen": 1016397824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034952858575727183, + "loss": 2.8292, + "theoretical_loss": 3.6435316335354555, + "tokens_seen": 1016463360 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349518555667001, + "loss": 2.6217, + "theoretical_loss": 3.6435093457287646, + "tokens_seen": 1016528896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003495085255767302, + "loss": 2.7067, + "theoretical_loss": 3.6434870597612337, + "tokens_seen": 1016594432 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494984954864594, + "loss": 2.7773, + "theoretical_loss": 3.6434647756325926, + "tokens_seen": 1016659968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034948846539618856, + "loss": 2.6261, + "theoretical_loss": 3.6434424933425715, + "tokens_seen": 1016725504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494784353059178, + "loss": 2.7, + "theoretical_loss": 3.6434202128909003, + "tokens_seen": 1016791040 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494684052156469, + "loss": 2.7586, + "theoretical_loss": 3.6433979342773077, + "tokens_seen": 1016856576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034945837512537616, + "loss": 2.8934, + "theoretical_loss": 3.643375657501525, + "tokens_seen": 1016922112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494483450351053, + "loss": 2.6247, + "theoretical_loss": 3.6433533825632813, + "tokens_seen": 1016987648 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494383149448345, + "loss": 2.8865, + "theoretical_loss": 3.6433311094623066, + "tokens_seen": 1017053184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494282848545637, + "loss": 2.6264, + "theoretical_loss": 3.643308838198332, + "tokens_seen": 1017118720 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003494182547642929, + "loss": 2.6972, + "theoretical_loss": 3.6432865687710865, + "tokens_seen": 1017184256 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034940822467402206, + "loss": 2.7361, + "theoretical_loss": 3.643264301180301, + "tokens_seen": 1017249792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493981945837513, + "loss": 2.6566, + "theoretical_loss": 3.643242035425705, + "tokens_seen": 1017315328 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493881644934804, + "loss": 2.6861, + "theoretical_loss": 3.64321977150703, + "tokens_seen": 1017380864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 568180, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6473095417022705, + "objective/train/theoretical_loss": 3.643197509424005, + "objective/train/tokens_used": 1037906400, + "theoretical_loss": 3.643197509424005, + "tokens_seen": 1017446400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034937813440320966, + "loss": 2.7746, + "theoretical_loss": 3.643197509424005, + "tokens_seen": 1017446400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493681043129388, + "loss": 2.8067, + "theoretical_loss": 3.6431752491763616, + "tokens_seen": 1017511936 + }, + { + "epoch": 0.31, + "learning_rate": 0.000349358074222668, + "loss": 2.6145, + "theoretical_loss": 3.64315299076383, + "tokens_seen": 1017577472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493480441323972, + "loss": 2.9663, + "theoretical_loss": 3.6431307341861396, + "tokens_seen": 1017643008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003493380140421264, + "loss": 2.7876, + "theoretical_loss": 3.6431084794430224, + "tokens_seen": 1017708544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034932798395185557, + "loss": 2.7973, + "theoretical_loss": 3.6430862265342085, + "tokens_seen": 1017774080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034931795386158475, + "loss": 2.5958, + "theoretical_loss": 3.643063975459429, + "tokens_seen": 1017839616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034930792377131393, + "loss": 2.8849, + "theoretical_loss": 3.643041726218414, + "tokens_seen": 1017905152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034929789368104316, + "loss": 2.7891, + "theoretical_loss": 3.643019478810895, + "tokens_seen": 1017970688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492878635907723, + "loss": 2.7187, + "theoretical_loss": 3.6429972332366027, + "tokens_seen": 1018036224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492778335005015, + "loss": 2.7684, + "theoretical_loss": 3.6429749894952677, + "tokens_seen": 1018101760 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034926780341023065, + "loss": 2.3443, + "theoretical_loss": 3.6429527475866212, + "tokens_seen": 1018167296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492577733199599, + "loss": 2.8958, + "theoretical_loss": 3.6429305075103944, + "tokens_seen": 1018232832 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034924774322968907, + "loss": 2.7228, + "theoretical_loss": 3.642908269266318, + "tokens_seen": 1018298368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034923771313941825, + "loss": 2.9386, + "theoretical_loss": 3.6428860328541233, + "tokens_seen": 1018363904 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034922768304914743, + "loss": 2.6324, + "theoretical_loss": 3.6428637982735417, + "tokens_seen": 1018429440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034921765295887667, + "loss": 2.6251, + "theoretical_loss": 3.642841565524305, + "tokens_seen": 1018494976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003492076228686058, + "loss": 2.6117, + "theoretical_loss": 3.6428193346061435, + "tokens_seen": 1018560512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034919759277833503, + "loss": 2.6193, + "theoretical_loss": 3.642797105518789, + "tokens_seen": 1018626048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034918756268806416, + "loss": 2.611, + "theoretical_loss": 3.6427748782619735, + "tokens_seen": 1018691584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491775325977934, + "loss": 2.5485, + "theoretical_loss": 3.6427526528354277, + "tokens_seen": 1018757120 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034916750250752257, + "loss": 2.8654, + "theoretical_loss": 3.642730429238884, + "tokens_seen": 1018822656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034915747241725175, + "loss": 2.8412, + "theoretical_loss": 3.6427082074720727, + "tokens_seen": 1018888192 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034914744232698093, + "loss": 2.5741, + "theoretical_loss": 3.6426859875347266, + "tokens_seen": 1018953728 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491374122367101, + "loss": 2.6678, + "theoretical_loss": 3.6426637694265773, + "tokens_seen": 1019019264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 568941, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.903310775756836, + "objective/train/theoretical_loss": 3.6426415531473566, + "objective/train/tokens_used": 1039544800, + "theoretical_loss": 3.6426415531473566, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003491273821464393, + "loss": 2.701, + "theoretical_loss": 3.6426415531473566, + "tokens_seen": 1019084800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034911735205616853, + "loss": 2.8147, + "theoretical_loss": 3.6426193386967958, + "tokens_seen": 1019150336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034910732196589766, + "loss": 2.5661, + "theoretical_loss": 3.642597126074627, + "tokens_seen": 1019215872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490972918756269, + "loss": 2.8054, + "theoretical_loss": 3.6425749152805826, + "tokens_seen": 1019281408 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034908726178535613, + "loss": 2.7761, + "theoretical_loss": 3.6425527063143948, + "tokens_seen": 1019346944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034907723169508526, + "loss": 2.6967, + "theoretical_loss": 3.6425304991757943, + "tokens_seen": 1019412480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490672016048145, + "loss": 2.9256, + "theoretical_loss": 3.6425082938645144, + "tokens_seen": 1019478016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490571715145436, + "loss": 2.9273, + "theoretical_loss": 3.642486090380287, + "tokens_seen": 1019543552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034904714142427285, + "loss": 2.6534, + "theoretical_loss": 3.642463888722845, + "tokens_seen": 1019609088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034903711133400203, + "loss": 2.8481, + "theoretical_loss": 3.6424416888919193, + "tokens_seen": 1019674624 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490270812437312, + "loss": 2.6814, + "theoretical_loss": 3.6424194908872436, + "tokens_seen": 1019740160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490170511534604, + "loss": 2.7889, + "theoretical_loss": 3.64239729470855, + "tokens_seen": 1019805696 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003490070210631896, + "loss": 2.8075, + "theoretical_loss": 3.6423751003555696, + "tokens_seen": 1019871232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034899699097291876, + "loss": 2.7066, + "theoretical_loss": 3.6423529078280366, + "tokens_seen": 1019936768 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348986960882648, + "loss": 2.6282, + "theoretical_loss": 3.642330717125683, + "tokens_seen": 1020002304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489769307923771, + "loss": 2.9868, + "theoretical_loss": 3.6423085282482415, + "tokens_seen": 1020067840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034896690070210636, + "loss": 2.8634, + "theoretical_loss": 3.6422863411954447, + "tokens_seen": 1020133376 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489568706118355, + "loss": 2.7896, + "theoretical_loss": 3.642264155967026, + "tokens_seen": 1020198912 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489468405215647, + "loss": 2.7245, + "theoretical_loss": 3.642241972562717, + "tokens_seen": 1020264448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489368104312939, + "loss": 2.7539, + "theoretical_loss": 3.642219790982251, + "tokens_seen": 1020329984 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489267803410231, + "loss": 2.3399, + "theoretical_loss": 3.6421976112253613, + "tokens_seen": 1020395520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034891675025075226, + "loss": 2.5317, + "theoretical_loss": 3.64217543329178, + "tokens_seen": 1020461056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003489067201604815, + "loss": 2.8228, + "theoretical_loss": 3.6421532571812416, + "tokens_seen": 1020526592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488966900702106, + "loss": 2.6505, + "theoretical_loss": 3.6421310828934783, + "tokens_seen": 1020592128 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034888665997993986, + "loss": 2.7931, + "theoretical_loss": 3.642108910428223, + "tokens_seen": 1020657664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 569860, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.728569507598877, + "objective/train/theoretical_loss": 3.6420867397852095, + "objective/train/tokens_used": 1041183200, + "theoretical_loss": 3.6420867397852095, + "tokens_seen": 1020723200 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348876629889669, + "loss": 2.6178, + "theoretical_loss": 3.6420867397852095, + "tokens_seen": 1020723200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488665997993982, + "loss": 2.7344, + "theoretical_loss": 3.6420645709641706, + "tokens_seen": 1020788736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488565697091274, + "loss": 2.82, + "theoretical_loss": 3.64204240396484, + "tokens_seen": 1020854272 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003488465396188566, + "loss": 2.7387, + "theoretical_loss": 3.6420202387869507, + "tokens_seen": 1020919808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034883650952858577, + "loss": 2.7402, + "theoretical_loss": 3.6419980754302363, + "tokens_seen": 1020985344 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034882647943831495, + "loss": 2.8513, + "theoretical_loss": 3.64197591389443, + "tokens_seen": 1021050880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034881644934804413, + "loss": 2.9306, + "theoretical_loss": 3.6419537541792657, + "tokens_seen": 1021116416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034880641925777336, + "loss": 2.7919, + "theoretical_loss": 3.6419315962844774, + "tokens_seen": 1021181952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487963891675025, + "loss": 2.6795, + "theoretical_loss": 3.6419094402097976, + "tokens_seen": 1021247488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487863590772317, + "loss": 2.6768, + "theoretical_loss": 3.6418872859549607, + "tokens_seen": 1021313024 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034877632898696085, + "loss": 2.8067, + "theoretical_loss": 3.6418651335197003, + "tokens_seen": 1021378560 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003487662988966901, + "loss": 2.7436, + "theoretical_loss": 3.641842982903751, + "tokens_seen": 1021444096 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034875626880641927, + "loss": 2.8722, + "theoretical_loss": 3.641820834106845, + "tokens_seen": 1021509632 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034874623871614845, + "loss": 2.8338, + "theoretical_loss": 3.641798687128718, + "tokens_seen": 1021575168 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034873620862587763, + "loss": 2.9625, + "theoretical_loss": 3.641776541969103, + "tokens_seen": 1021640704 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034872617853560687, + "loss": 2.5987, + "theoretical_loss": 3.641754398627734, + "tokens_seen": 1021706240 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348716148445336, + "loss": 2.882, + "theoretical_loss": 3.6417322571043456, + "tokens_seen": 1021771776 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034870611835506523, + "loss": 2.762, + "theoretical_loss": 3.641710117398672, + "tokens_seen": 1021837312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034869608826479436, + "loss": 2.5842, + "theoretical_loss": 3.641687979510446, + "tokens_seen": 1021902848 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486860581745236, + "loss": 2.7492, + "theoretical_loss": 3.6416658434394042, + "tokens_seen": 1021968384 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034867602808425277, + "loss": 2.8195, + "theoretical_loss": 3.6416437091852787, + "tokens_seen": 1022033920 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034866599799398195, + "loss": 2.9394, + "theoretical_loss": 3.6416215767478057, + "tokens_seen": 1022099456 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034865596790371113, + "loss": 2.5965, + "theoretical_loss": 3.641599446126718, + "tokens_seen": 1022164992 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486459378134403, + "loss": 2.7538, + "theoretical_loss": 3.6415773173217514, + "tokens_seen": 1022230528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486359077231695, + "loss": 2.5377, + "theoretical_loss": 3.6415551903326397, + "tokens_seen": 1022296064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 570478, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2970290184020996, + "objective/train/theoretical_loss": 3.641533065159118, + "objective/train/tokens_used": 1042821600, + "theoretical_loss": 3.641533065159118, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034862587763289873, + "loss": 2.6148, + "theoretical_loss": 3.641533065159118, + "tokens_seen": 1022361600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034861584754262786, + "loss": 2.8847, + "theoretical_loss": 3.6415109418009206, + "tokens_seen": 1022427136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003486058174523571, + "loss": 2.7738, + "theoretical_loss": 3.6414888202577824, + "tokens_seen": 1022492672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485957873620862, + "loss": 2.9431, + "theoretical_loss": 3.6414667005294374, + "tokens_seen": 1022558208 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034858575727181546, + "loss": 2.7978, + "theoretical_loss": 3.641444582615622, + "tokens_seen": 1022623744 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034857572718154464, + "loss": 2.7741, + "theoretical_loss": 3.6414224665160697, + "tokens_seen": 1022689280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485656970912738, + "loss": 2.6496, + "theoretical_loss": 3.641400352230516, + "tokens_seen": 1022754816 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348555667001003, + "loss": 2.6782, + "theoretical_loss": 3.6413782397586956, + "tokens_seen": 1022820352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034854563691073223, + "loss": 2.6638, + "theoretical_loss": 3.641356129100344, + "tokens_seen": 1022885888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034853560682046136, + "loss": 2.7403, + "theoretical_loss": 3.6413340202551954, + "tokens_seen": 1022951424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485255767301906, + "loss": 2.8024, + "theoretical_loss": 3.641311913222986, + "tokens_seen": 1023016960 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003485155466399197, + "loss": 2.6215, + "theoretical_loss": 3.6412898080034504, + "tokens_seen": 1023082496 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034850551654964896, + "loss": 2.8317, + "theoretical_loss": 3.6412677045963244, + "tokens_seen": 1023148032 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034849548645937814, + "loss": 2.7537, + "theoretical_loss": 3.6412456030013427, + "tokens_seen": 1023213568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484854563691073, + "loss": 2.6645, + "theoretical_loss": 3.6412235032182414, + "tokens_seen": 1023279104 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484754262788365, + "loss": 2.4196, + "theoretical_loss": 3.6412014052467554, + "tokens_seen": 1023344640 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484653961885657, + "loss": 2.4108, + "theoretical_loss": 3.64117930908662, + "tokens_seen": 1023410176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034845536609829486, + "loss": 2.8733, + "theoretical_loss": 3.6411572147375715, + "tokens_seen": 1023475712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484453360080241, + "loss": 2.6105, + "theoretical_loss": 3.6411351221993447, + "tokens_seen": 1023541248 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484353059177532, + "loss": 2.6572, + "theoretical_loss": 3.6411130314716758, + "tokens_seen": 1023606784 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034842527582748246, + "loss": 2.5466, + "theoretical_loss": 3.6410909425542997, + "tokens_seen": 1023672320 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034841524573721164, + "loss": 2.493, + "theoretical_loss": 3.641068855446953, + "tokens_seen": 1023737856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003484052156469408, + "loss": 2.6693, + "theoretical_loss": 3.641046770149372, + "tokens_seen": 1023803392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034839518555667, + "loss": 2.8194, + "theoretical_loss": 3.641024686661291, + "tokens_seen": 1023868928 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483851554663992, + "loss": 2.6758, + "theoretical_loss": 3.6410026049824475, + "tokens_seen": 1023934464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 571810, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.663266897201538, + "objective/train/theoretical_loss": 3.640980525112577, + "objective/train/tokens_used": 1044460000, + "theoretical_loss": 3.640980525112577, + "tokens_seen": 1024000000 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034837512537612837, + "loss": 2.5753, + "theoretical_loss": 3.640980525112577, + "tokens_seen": 1024000000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483650952858576, + "loss": 2.7885, + "theoretical_loss": 3.640958447051415, + "tokens_seen": 1024065536 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034835506519558673, + "loss": 2.5495, + "theoretical_loss": 3.6409363707986975, + "tokens_seen": 1024131072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034834503510531597, + "loss": 2.8806, + "theoretical_loss": 3.6409142963541616, + "tokens_seen": 1024196608 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034833500501504515, + "loss": 2.8696, + "theoretical_loss": 3.6408922237175427, + "tokens_seen": 1024262144 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034832497492477433, + "loss": 2.6233, + "theoretical_loss": 3.640870152888578, + "tokens_seen": 1024327680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034831494483450356, + "loss": 2.7639, + "theoretical_loss": 3.6408480838670023, + "tokens_seen": 1024393216 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003483049147442327, + "loss": 2.8172, + "theoretical_loss": 3.6408260166525537, + "tokens_seen": 1024458752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482948846539619, + "loss": 2.7901, + "theoretical_loss": 3.6408039512449677, + "tokens_seen": 1024524288 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034828485456369105, + "loss": 2.7927, + "theoretical_loss": 3.6407818876439806, + "tokens_seen": 1024589824 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482748244734203, + "loss": 2.8879, + "theoretical_loss": 3.6407598258493294, + "tokens_seen": 1024655360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034826479438314947, + "loss": 2.7932, + "theoretical_loss": 3.6407377658607505, + "tokens_seen": 1024720896 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034825476429287865, + "loss": 2.6392, + "theoretical_loss": 3.640715707677981, + "tokens_seen": 1024786432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034824473420260783, + "loss": 2.8363, + "theoretical_loss": 3.6406936513007566, + "tokens_seen": 1024851968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034823470411233707, + "loss": 2.6225, + "theoretical_loss": 3.640671596728815, + "tokens_seen": 1024917504 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003482246740220662, + "loss": 2.8723, + "theoretical_loss": 3.6406495439618927, + "tokens_seen": 1024983040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034821464393179543, + "loss": 2.6802, + "theoretical_loss": 3.6406274929997267, + "tokens_seen": 1025048576 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034820461384152456, + "loss": 2.6781, + "theoretical_loss": 3.6406054438420536, + "tokens_seen": 1025114112 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481945837512538, + "loss": 2.7748, + "theoretical_loss": 3.6405833964886103, + "tokens_seen": 1025179648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034818455366098297, + "loss": 2.6838, + "theoretical_loss": 3.6405613509391346, + "tokens_seen": 1025245184 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034817452357071215, + "loss": 2.782, + "theoretical_loss": 3.640539307193363, + "tokens_seen": 1025310720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034816449348044133, + "loss": 2.7386, + "theoretical_loss": 3.640517265251033, + "tokens_seen": 1025376256 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481544633901705, + "loss": 2.9111, + "theoretical_loss": 3.6404952251118816, + "tokens_seen": 1025441792 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481444332998997, + "loss": 2.5696, + "theoretical_loss": 3.6404731867756457, + "tokens_seen": 1025507328 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034813440320962893, + "loss": 2.5349, + "theoretical_loss": 3.640451150242063, + "tokens_seen": 1025572864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 572959, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5865912437438965, + "objective/train/theoretical_loss": 3.6404291155108712, + "objective/train/tokens_used": 1046098400, + "theoretical_loss": 3.6404291155108712, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034812437311935806, + "loss": 2.5791, + "theoretical_loss": 3.6404291155108712, + "tokens_seen": 1025638400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481143430290873, + "loss": 2.7193, + "theoretical_loss": 3.640407082581807, + "tokens_seen": 1025703936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003481043129388164, + "loss": 2.7651, + "theoretical_loss": 3.6403850514546083, + "tokens_seen": 1025769472 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034809428284854566, + "loss": 2.6543, + "theoretical_loss": 3.6403630221290126, + "tokens_seen": 1025835008 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034808425275827484, + "loss": 2.7465, + "theoretical_loss": 3.640340994604758, + "tokens_seen": 1025900544 + }, + { + "epoch": 0.31, + "learning_rate": 0.000348074222668004, + "loss": 2.5764, + "theoretical_loss": 3.640318968881581, + "tokens_seen": 1025966080 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480641925777332, + "loss": 2.74, + "theoretical_loss": 3.6402969449592195, + "tokens_seen": 1026031616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034805416248746243, + "loss": 2.4407, + "theoretical_loss": 3.6402749228374125, + "tokens_seen": 1026097152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034804413239719156, + "loss": 2.5572, + "theoretical_loss": 3.6402529025158965, + "tokens_seen": 1026162688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480341023069208, + "loss": 2.9396, + "theoretical_loss": 3.64023088399441, + "tokens_seen": 1026228224 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003480240722166499, + "loss": 2.6433, + "theoretical_loss": 3.640208867272691, + "tokens_seen": 1026293760 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034801404212637916, + "loss": 2.8696, + "theoretical_loss": 3.6401868523504772, + "tokens_seen": 1026359296 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034800401203610834, + "loss": 2.6728, + "theoretical_loss": 3.640164839227506, + "tokens_seen": 1026424832 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479939819458375, + "loss": 2.6425, + "theoretical_loss": 3.640142827903517, + "tokens_seen": 1026490368 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479839518555667, + "loss": 2.7012, + "theoretical_loss": 3.6401208183782474, + "tokens_seen": 1026555904 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479739217652959, + "loss": 2.7336, + "theoretical_loss": 3.640098810651435, + "tokens_seen": 1026621440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034796389167502506, + "loss": 2.8522, + "theoretical_loss": 3.6400768047228187, + "tokens_seen": 1026686976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479538615847543, + "loss": 2.7545, + "theoretical_loss": 3.6400548005921367, + "tokens_seen": 1026752512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034794383149448343, + "loss": 2.6936, + "theoretical_loss": 3.6400327982591274, + "tokens_seen": 1026818048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034793380140421266, + "loss": 2.7252, + "theoretical_loss": 3.640010797723529, + "tokens_seen": 1026883584 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034792377131394184, + "loss": 2.9514, + "theoretical_loss": 3.6399887989850805, + "tokens_seen": 1026949120 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347913741223671, + "loss": 2.6321, + "theoretical_loss": 3.63996680204352, + "tokens_seen": 1027014656 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003479037111334002, + "loss": 2.6391, + "theoretical_loss": 3.639944806898585, + "tokens_seen": 1027080192 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478936810431294, + "loss": 2.6399, + "theoretical_loss": 3.639922813550016, + "tokens_seen": 1027145728 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034788365095285857, + "loss": 2.8079, + "theoretical_loss": 3.6399008219975513, + "tokens_seen": 1027211264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 573554, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3572163581848145, + "objective/train/theoretical_loss": 3.639878832240929, + "objective/train/tokens_used": 1047736800, + "theoretical_loss": 3.639878832240929, + "tokens_seen": 1027276800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478736208625878, + "loss": 2.6548, + "theoretical_loss": 3.639878832240929, + "tokens_seen": 1027276800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034786359077231693, + "loss": 2.6283, + "theoretical_loss": 3.6398568442798878, + "tokens_seen": 1027342336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034785356068204617, + "loss": 2.7042, + "theoretical_loss": 3.6398348581141677, + "tokens_seen": 1027407872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478435305917753, + "loss": 2.6247, + "theoretical_loss": 3.639812873743506, + "tokens_seen": 1027473408 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034783350050150453, + "loss": 2.6911, + "theoretical_loss": 3.6397908911676424, + "tokens_seen": 1027538944 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478234704112337, + "loss": 2.8631, + "theoretical_loss": 3.6397689103863167, + "tokens_seen": 1027604480 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003478134403209629, + "loss": 2.7386, + "theoretical_loss": 3.6397469313992667, + "tokens_seen": 1027670016 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034780341023069207, + "loss": 2.7186, + "theoretical_loss": 3.6397249542062324, + "tokens_seen": 1027735552 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034779338014042125, + "loss": 2.8693, + "theoretical_loss": 3.6397029788069526, + "tokens_seen": 1027801088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034778335005015043, + "loss": 2.5519, + "theoretical_loss": 3.639681005201167, + "tokens_seen": 1027866624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034777331995987967, + "loss": 2.6078, + "theoretical_loss": 3.6396590333886136, + "tokens_seen": 1027932160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477632898696088, + "loss": 2.5822, + "theoretical_loss": 3.639637063369033, + "tokens_seen": 1027997696 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034775325977933803, + "loss": 2.8317, + "theoretical_loss": 3.6396150951421644, + "tokens_seen": 1028063232 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477432296890672, + "loss": 2.6957, + "theoretical_loss": 3.639593128707747, + "tokens_seen": 1028128768 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477331995987964, + "loss": 2.6182, + "theoretical_loss": 3.6395711640655204, + "tokens_seen": 1028194304 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003477231695085256, + "loss": 2.5697, + "theoretical_loss": 3.639549201215224, + "tokens_seen": 1028259840 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034771313941825476, + "loss": 2.4462, + "theoretical_loss": 3.639527240156598, + "tokens_seen": 1028325376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034770310932798394, + "loss": 2.7429, + "theoretical_loss": 3.6395052808893813, + "tokens_seen": 1028390912 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034769307923771317, + "loss": 2.8586, + "theoretical_loss": 3.639483323413314, + "tokens_seen": 1028456448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476830491474423, + "loss": 2.8388, + "theoretical_loss": 3.6394613677281358, + "tokens_seen": 1028521984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034767301905717153, + "loss": 2.7639, + "theoretical_loss": 3.6394394138335864, + "tokens_seen": 1028587520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034766298896690066, + "loss": 2.7978, + "theoretical_loss": 3.639417461729406, + "tokens_seen": 1028653056 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476529588766299, + "loss": 2.6436, + "theoretical_loss": 3.6393955114153345, + "tokens_seen": 1028718592 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476429287863591, + "loss": 2.7939, + "theoretical_loss": 3.6393735628911115, + "tokens_seen": 1028784128 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034763289869608826, + "loss": 2.563, + "theoretical_loss": 3.639351616156478, + "tokens_seen": 1028849664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 574441, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6161293983459473, + "objective/train/theoretical_loss": 3.639329671211173, + "objective/train/tokens_used": 1049375200, + "theoretical_loss": 3.639329671211173, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034762286860581744, + "loss": 2.5903, + "theoretical_loss": 3.639329671211173, + "tokens_seen": 1028915200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476128385155466, + "loss": 2.5422, + "theoretical_loss": 3.639307728054937, + "tokens_seen": 1028980736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003476028084252758, + "loss": 2.5475, + "theoretical_loss": 3.639285786687511, + "tokens_seen": 1029046272 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034759277833500504, + "loss": 3.0258, + "theoretical_loss": 3.6392638471086336, + "tokens_seen": 1029111808 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475827482447342, + "loss": 2.5871, + "theoretical_loss": 3.639241909318047, + "tokens_seen": 1029177344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475727181544634, + "loss": 2.6932, + "theoretical_loss": 3.6392199733154897, + "tokens_seen": 1029242880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034756268806419263, + "loss": 2.6994, + "theoretical_loss": 3.6391980391007035, + "tokens_seen": 1029308416 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034755265797392176, + "loss": 2.6649, + "theoretical_loss": 3.639176106673429, + "tokens_seen": 1029373952 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347542627883651, + "loss": 2.4648, + "theoretical_loss": 3.639154176033406, + "tokens_seen": 1029439488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475325977933801, + "loss": 2.7083, + "theoretical_loss": 3.639132247180375, + "tokens_seen": 1029505024 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034752256770310936, + "loss": 2.6017, + "theoretical_loss": 3.639110320114077, + "tokens_seen": 1029570560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034751253761283854, + "loss": 2.6896, + "theoretical_loss": 3.6390883948342534, + "tokens_seen": 1029636096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003475025075225677, + "loss": 2.811, + "theoretical_loss": 3.6390664713406435, + "tokens_seen": 1029701632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474924774322969, + "loss": 2.5898, + "theoretical_loss": 3.639044549632989, + "tokens_seen": 1029767168 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474824473420261, + "loss": 2.7057, + "theoretical_loss": 3.6390226297110306, + "tokens_seen": 1029832704 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034747241725175527, + "loss": 2.2744, + "theoretical_loss": 3.6390007115745098, + "tokens_seen": 1029898240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474623871614845, + "loss": 2.6949, + "theoretical_loss": 3.6389787952231663, + "tokens_seen": 1029963776 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034745235707121363, + "loss": 2.5461, + "theoretical_loss": 3.6389568806567416, + "tokens_seen": 1030029312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034744232698094286, + "loss": 2.6111, + "theoretical_loss": 3.638934967874978, + "tokens_seen": 1030094848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034743229689067204, + "loss": 2.8444, + "theoretical_loss": 3.638913056877615, + "tokens_seen": 1030160384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474222668004012, + "loss": 2.7156, + "theoretical_loss": 3.6388911476643946, + "tokens_seen": 1030225920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474122367101304, + "loss": 2.7605, + "theoretical_loss": 3.6388692402350578, + "tokens_seen": 1030291456 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003474022066198596, + "loss": 2.578, + "theoretical_loss": 3.6388473345893457, + "tokens_seen": 1030356992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034739217652958877, + "loss": 2.6943, + "theoretical_loss": 3.638825430727, + "tokens_seen": 1030422528 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347382146439318, + "loss": 3.0443, + "theoretical_loss": 3.6388035286477622, + "tokens_seen": 1030488064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 574891, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2433831691741943, + "objective/train/theoretical_loss": 3.6387816283513734, + "objective/train/tokens_used": 1051013600, + "theoretical_loss": 3.6387816283513734, + "tokens_seen": 1030553600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034737211634904713, + "loss": 2.8729, + "theoretical_loss": 3.6387816283513734, + "tokens_seen": 1030553600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034736208625877637, + "loss": 2.4595, + "theoretical_loss": 3.6387597298375756, + "tokens_seen": 1030619136 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473520561685055, + "loss": 2.7197, + "theoretical_loss": 3.63873783310611, + "tokens_seen": 1030684672 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034734202607823473, + "loss": 2.8291, + "theoretical_loss": 3.638715938156718, + "tokens_seen": 1030750208 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473319959879639, + "loss": 2.9111, + "theoretical_loss": 3.638694044989142, + "tokens_seen": 1030815744 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003473219658976931, + "loss": 2.5698, + "theoretical_loss": 3.638672153603123, + "tokens_seen": 1030881280 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034731193580742227, + "loss": 2.6734, + "theoretical_loss": 3.638650263998403, + "tokens_seen": 1030946816 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034730190571715145, + "loss": 2.8147, + "theoretical_loss": 3.638628376174724, + "tokens_seen": 1031012352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034729187562688063, + "loss": 2.7605, + "theoretical_loss": 3.638606490131828, + "tokens_seen": 1031077888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034728184553660987, + "loss": 2.5385, + "theoretical_loss": 3.6385846058694558, + "tokens_seen": 1031143424 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347271815446339, + "loss": 2.8986, + "theoretical_loss": 3.638562723387351, + "tokens_seen": 1031208960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034726178535606823, + "loss": 2.7973, + "theoretical_loss": 3.638540842685255, + "tokens_seen": 1031274496 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472517552657974, + "loss": 2.6124, + "theoretical_loss": 3.63851896376291, + "tokens_seen": 1031340032 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472417251755266, + "loss": 2.5257, + "theoretical_loss": 3.6384970866200574, + "tokens_seen": 1031405568 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003472316950852558, + "loss": 2.7745, + "theoretical_loss": 3.6384752112564405, + "tokens_seen": 1031471104 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034722166499498496, + "loss": 2.8269, + "theoretical_loss": 3.638453337671801, + "tokens_seen": 1031536640 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034721163490471414, + "loss": 2.6715, + "theoretical_loss": 3.638431465865881, + "tokens_seen": 1031602176 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034720160481444337, + "loss": 2.8328, + "theoretical_loss": 3.6384095958384237, + "tokens_seen": 1031667712 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471915747241725, + "loss": 2.7759, + "theoretical_loss": 3.638387727589171, + "tokens_seen": 1031733248 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034718154463390173, + "loss": 3.0587, + "theoretical_loss": 3.6383658611178653, + "tokens_seen": 1031798784 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034717151454363086, + "loss": 2.4092, + "theoretical_loss": 3.6383439964242488, + "tokens_seen": 1031864320 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471614844533601, + "loss": 2.5852, + "theoretical_loss": 3.638322133508065, + "tokens_seen": 1031929856 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471514543630893, + "loss": 2.6847, + "theoretical_loss": 3.6383002723690563, + "tokens_seen": 1031995392 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034714142427281846, + "loss": 2.7011, + "theoretical_loss": 3.6382784130069643, + "tokens_seen": 1032060928 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034713139418254764, + "loss": 2.8238, + "theoretical_loss": 3.6382565554215334, + "tokens_seen": 1032126464 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 576151, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7170889377593994, + "objective/train/theoretical_loss": 3.6382346996125055, + "objective/train/tokens_used": 1052652000, + "theoretical_loss": 3.6382346996125055, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003471213640922768, + "loss": 2.6599, + "theoretical_loss": 3.6382346996125055, + "tokens_seen": 1032192000 + }, + { + "epoch": 0.31, + "learning_rate": 0.000347111334002006, + "loss": 2.6971, + "theoretical_loss": 3.638212845579623, + "tokens_seen": 1032257536 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034710130391173524, + "loss": 2.6737, + "theoretical_loss": 3.63819099332263, + "tokens_seen": 1032323072 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034709127382146436, + "loss": 2.8611, + "theoretical_loss": 3.6381691428412686, + "tokens_seen": 1032388608 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470812437311936, + "loss": 2.5908, + "theoretical_loss": 3.6381472941352815, + "tokens_seen": 1032454144 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470712136409228, + "loss": 2.7545, + "theoretical_loss": 3.638125447204413, + "tokens_seen": 1032519680 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034706118355065196, + "loss": 2.8943, + "theoretical_loss": 3.6381036020484054, + "tokens_seen": 1032585216 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034705115346038114, + "loss": 2.7134, + "theoretical_loss": 3.638081758667002, + "tokens_seen": 1032650752 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470411233701103, + "loss": 2.6238, + "theoretical_loss": 3.6380599170599464, + "tokens_seen": 1032716288 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470310932798395, + "loss": 2.7772, + "theoretical_loss": 3.6380380772269807, + "tokens_seen": 1032781824 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034702106318956874, + "loss": 2.7171, + "theoretical_loss": 3.6380162391678494, + "tokens_seen": 1032847360 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034701103309929787, + "loss": 2.7668, + "theoretical_loss": 3.637994402882296, + "tokens_seen": 1032912896 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003470010030090271, + "loss": 2.5211, + "theoretical_loss": 3.637972568370063, + "tokens_seen": 1032978432 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034699097291875623, + "loss": 2.7032, + "theoretical_loss": 3.6379507356308944, + "tokens_seen": 1033043968 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034698094282848547, + "loss": 2.818, + "theoretical_loss": 3.637928904664534, + "tokens_seen": 1033109504 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034697091273821465, + "loss": 2.4894, + "theoretical_loss": 3.637907075470725, + "tokens_seen": 1033175040 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034696088264794383, + "loss": 2.7589, + "theoretical_loss": 3.637885248049211, + "tokens_seen": 1033240576 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346950852557673, + "loss": 2.5823, + "theoretical_loss": 3.6378634223997364, + "tokens_seen": 1033306112 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034694082246740224, + "loss": 2.6596, + "theoretical_loss": 3.6378415985220443, + "tokens_seen": 1033371648 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034693079237713137, + "loss": 2.7211, + "theoretical_loss": 3.6378197764158786, + "tokens_seen": 1033437184 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003469207622868606, + "loss": 2.4441, + "theoretical_loss": 3.637797956080983, + "tokens_seen": 1033502720 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034691073219658973, + "loss": 2.701, + "theoretical_loss": 3.6377761375171023, + "tokens_seen": 1033568256 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034690070210631897, + "loss": 2.5034, + "theoretical_loss": 3.637754320723979, + "tokens_seen": 1033633792 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034689067201604815, + "loss": 2.615, + "theoretical_loss": 3.6377325057013588, + "tokens_seen": 1033699328 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034688064192577733, + "loss": 2.6485, + "theoretical_loss": 3.6377106924489846, + "tokens_seen": 1033764864 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 577510, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.366300106048584, + "objective/train/theoretical_loss": 3.637688880966601, + "objective/train/tokens_used": 1054290400, + "theoretical_loss": 3.637688880966601, + "tokens_seen": 1033830400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468706118355065, + "loss": 2.4252, + "theoretical_loss": 3.637688880966601, + "tokens_seen": 1033830400 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468605817452357, + "loss": 2.768, + "theoretical_loss": 3.6376670712539516, + "tokens_seen": 1033895936 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468505516549649, + "loss": 2.7405, + "theoretical_loss": 3.6376452633107816, + "tokens_seen": 1033961472 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468405215646941, + "loss": 2.7376, + "theoretical_loss": 3.637623457136835, + "tokens_seen": 1034027008 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003468304914744233, + "loss": 2.6507, + "theoretical_loss": 3.6376016527318558, + "tokens_seen": 1034092544 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034682046138415247, + "loss": 2.808, + "theoretical_loss": 3.6375798500955887, + "tokens_seen": 1034158080 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034681043129388165, + "loss": 2.5972, + "theoretical_loss": 3.6375580492277777, + "tokens_seen": 1034223616 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034680040120361083, + "loss": 2.8287, + "theoretical_loss": 3.637536250128168, + "tokens_seen": 1034289152 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034679037111334007, + "loss": 2.6991, + "theoretical_loss": 3.6375144527965038, + "tokens_seen": 1034354688 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467803410230692, + "loss": 2.8229, + "theoretical_loss": 3.6374926572325297, + "tokens_seen": 1034420224 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034677031093279843, + "loss": 2.7673, + "theoretical_loss": 3.63747086343599, + "tokens_seen": 1034485760 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467602808425276, + "loss": 2.7937, + "theoretical_loss": 3.63744907140663, + "tokens_seen": 1034551296 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467502507522568, + "loss": 2.7354, + "theoretical_loss": 3.637427281144195, + "tokens_seen": 1034616832 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346740220661986, + "loss": 2.7832, + "theoretical_loss": 3.637405492648428, + "tokens_seen": 1034682368 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034673019057171516, + "loss": 2.7563, + "theoretical_loss": 3.637383705919076, + "tokens_seen": 1034747904 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034672016048144434, + "loss": 2.7772, + "theoretical_loss": 3.637361920955882, + "tokens_seen": 1034813440 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034671013039117357, + "loss": 2.7117, + "theoretical_loss": 3.637340137758592, + "tokens_seen": 1034878976 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003467001003009027, + "loss": 2.6319, + "theoretical_loss": 3.6373183563269516, + "tokens_seen": 1034944512 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034669007021063193, + "loss": 2.608, + "theoretical_loss": 3.637296576660705, + "tokens_seen": 1035010048 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034668004012036106, + "loss": 2.5727, + "theoretical_loss": 3.6372747987595977, + "tokens_seen": 1035075584 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466700100300903, + "loss": 2.7038, + "theoretical_loss": 3.6372530226233746, + "tokens_seen": 1035141120 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466599799398195, + "loss": 2.7065, + "theoretical_loss": 3.637231248251781, + "tokens_seen": 1035206656 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034664994984954866, + "loss": 2.9293, + "theoretical_loss": 3.6372094756445623, + "tokens_seen": 1035272192 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034663991975927784, + "loss": 2.7916, + "theoretical_loss": 3.6371877048014642, + "tokens_seen": 1035337728 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346629889669007, + "loss": 2.5348, + "theoretical_loss": 3.6371659357222317, + "tokens_seen": 1035403264 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 578118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6748743057250977, + "objective/train/theoretical_loss": 3.6371441684066097, + "objective/train/tokens_used": 1055928800, + "theoretical_loss": 3.6371441684066097, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003466198595787362, + "loss": 2.7133, + "theoretical_loss": 3.6371441684066097, + "tokens_seen": 1035468800 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034660982948846544, + "loss": 2.5092, + "theoretical_loss": 3.637122402854345, + "tokens_seen": 1035534336 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034659979939819456, + "loss": 2.6481, + "theoretical_loss": 3.6371006390651828, + "tokens_seen": 1035599872 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465897693079238, + "loss": 2.7212, + "theoretical_loss": 3.637078877038868, + "tokens_seen": 1035665408 + }, + { + "epoch": 0.31, + "learning_rate": 0.000346579739217653, + "loss": 2.4035, + "theoretical_loss": 3.6370571167751473, + "tokens_seen": 1035730944 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034656970912738216, + "loss": 2.4784, + "theoretical_loss": 3.6370353582737653, + "tokens_seen": 1035796480 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034655967903711134, + "loss": 2.5488, + "theoretical_loss": 3.6370136015344685, + "tokens_seen": 1035862016 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465496489468405, + "loss": 2.713, + "theoretical_loss": 3.6369918465570024, + "tokens_seen": 1035927552 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465396188565697, + "loss": 2.7944, + "theoretical_loss": 3.6369700933411133, + "tokens_seen": 1035993088 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034652958876629894, + "loss": 2.6355, + "theoretical_loss": 3.636948341886547, + "tokens_seen": 1036058624 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034651955867602807, + "loss": 2.6931, + "theoretical_loss": 3.636926592193049, + "tokens_seen": 1036124160 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003465095285857573, + "loss": 2.761, + "theoretical_loss": 3.636904844260366, + "tokens_seen": 1036189696 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034649949849548643, + "loss": 2.7571, + "theoretical_loss": 3.636883098088244, + "tokens_seen": 1036255232 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034648946840521567, + "loss": 2.7743, + "theoretical_loss": 3.6368613536764283, + "tokens_seen": 1036320768 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034647943831494485, + "loss": 2.2489, + "theoretical_loss": 3.6368396110246666, + "tokens_seen": 1036386304 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034646940822467403, + "loss": 2.7951, + "theoretical_loss": 3.6368178701327034, + "tokens_seen": 1036451840 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464593781344032, + "loss": 2.5367, + "theoretical_loss": 3.6367961310002865, + "tokens_seen": 1036517376 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034644934804413244, + "loss": 2.635, + "theoretical_loss": 3.636774393627162, + "tokens_seen": 1036582912 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034643931795386157, + "loss": 2.563, + "theoretical_loss": 3.6367526580130756, + "tokens_seen": 1036648448 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003464292878635908, + "loss": 2.4748, + "theoretical_loss": 3.6367309241577743, + "tokens_seen": 1036713984 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034641925777331993, + "loss": 2.7516, + "theoretical_loss": 3.636709192061004, + "tokens_seen": 1036779520 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034640922768304917, + "loss": 2.6484, + "theoretical_loss": 3.636687461722512, + "tokens_seen": 1036845056 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034639919759277835, + "loss": 2.8035, + "theoretical_loss": 3.6366657331420447, + "tokens_seen": 1036910592 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034638916750250753, + "loss": 2.6139, + "theoretical_loss": 3.636644006319349, + "tokens_seen": 1036976128 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463791374122367, + "loss": 2.7835, + "theoretical_loss": 3.636622281254171, + "tokens_seen": 1037041664 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 579340, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.744521379470825, + "objective/train/theoretical_loss": 3.6366005579462577, + "objective/train/tokens_used": 1057567200, + "theoretical_loss": 3.6366005579462577, + "tokens_seen": 1037107200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463691073219659, + "loss": 2.6837, + "theoretical_loss": 3.6366005579462577, + "tokens_seen": 1037107200 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463590772316951, + "loss": 2.7886, + "theoretical_loss": 3.6365788363953566, + "tokens_seen": 1037172736 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463490471414243, + "loss": 2.566, + "theoretical_loss": 3.636557116601214, + "tokens_seen": 1037238272 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034633901705115344, + "loss": 2.6344, + "theoretical_loss": 3.636535398563576, + "tokens_seen": 1037303808 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034632898696088267, + "loss": 2.6984, + "theoretical_loss": 3.6365136822821915, + "tokens_seen": 1037369344 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003463189568706118, + "loss": 2.5895, + "theoretical_loss": 3.636491967756806, + "tokens_seen": 1037434880 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034630892678034103, + "loss": 2.8962, + "theoretical_loss": 3.6364702549871675, + "tokens_seen": 1037500416 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462988966900702, + "loss": 2.7219, + "theoretical_loss": 3.6364485439730223, + "tokens_seen": 1037565952 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462888665997994, + "loss": 2.6679, + "theoretical_loss": 3.6364268347141184, + "tokens_seen": 1037631488 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462788365095286, + "loss": 2.7464, + "theoretical_loss": 3.6364051272102027, + "tokens_seen": 1037697024 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462688064192578, + "loss": 2.825, + "theoretical_loss": 3.636383421461022, + "tokens_seen": 1037762560 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034625877632898694, + "loss": 2.603, + "theoretical_loss": 3.6363617174663245, + "tokens_seen": 1037828096 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462487462387162, + "loss": 2.7439, + "theoretical_loss": 3.6363400152258576, + "tokens_seen": 1037893632 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462387161484453, + "loss": 2.6634, + "theoretical_loss": 3.636318314739368, + "tokens_seen": 1037959168 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034622868605817454, + "loss": 2.6705, + "theoretical_loss": 3.6362966160066037, + "tokens_seen": 1038024704 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462186559679037, + "loss": 2.5983, + "theoretical_loss": 3.6362749190273123, + "tokens_seen": 1038090240 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003462086258776329, + "loss": 2.7591, + "theoretical_loss": 3.6362532238012415, + "tokens_seen": 1038155776 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461985957873621, + "loss": 2.3577, + "theoretical_loss": 3.6362315303281383, + "tokens_seen": 1038221312 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034618856569709126, + "loss": 2.5005, + "theoretical_loss": 3.6362098386077513, + "tokens_seen": 1038286848 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034617853560682044, + "loss": 2.6127, + "theoretical_loss": 3.636188148639828, + "tokens_seen": 1038352384 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461685055165497, + "loss": 2.8448, + "theoretical_loss": 3.636166460424116, + "tokens_seen": 1038417920 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461584754262788, + "loss": 2.3289, + "theoretical_loss": 3.636144773960363, + "tokens_seen": 1038483456 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034614844533600804, + "loss": 2.5375, + "theoretical_loss": 3.6361230892483176, + "tokens_seen": 1038548992 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034613841524573717, + "loss": 2.7036, + "theoretical_loss": 3.636101406287727, + "tokens_seen": 1038614528 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461283851554664, + "loss": 2.4236, + "theoretical_loss": 3.63607972507834, + "tokens_seen": 1038680064 + }, + { + "epoch": 0.31, + "objective/train/docs_used": 579796, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.493173837661743, + "objective/train/theoretical_loss": 3.6360580456199036, + "objective/train/tokens_used": 1059205600, + "theoretical_loss": 3.6360580456199036, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003461183550651956, + "loss": 2.5022, + "theoretical_loss": 3.6360580456199036, + "tokens_seen": 1038745600 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034610832497492476, + "loss": 2.841, + "theoretical_loss": 3.636036367912167, + "tokens_seen": 1038811136 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034609829488465395, + "loss": 2.7636, + "theoretical_loss": 3.6360146919548786, + "tokens_seen": 1038876672 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460882647943832, + "loss": 2.8077, + "theoretical_loss": 3.6359930177477855, + "tokens_seen": 1038942208 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034607823470411236, + "loss": 2.6653, + "theoretical_loss": 3.635971345290637, + "tokens_seen": 1039007744 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034606820461384154, + "loss": 2.5918, + "theoretical_loss": 3.635949674583181, + "tokens_seen": 1039073280 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460581745235707, + "loss": 2.6781, + "theoretical_loss": 3.635928005625166, + "tokens_seen": 1039138816 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460481444332999, + "loss": 2.6605, + "theoretical_loss": 3.6359063384163397, + "tokens_seen": 1039204352 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034603811434302914, + "loss": 2.6383, + "theoretical_loss": 3.635884672956452, + "tokens_seen": 1039269888 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034602808425275827, + "loss": 2.7481, + "theoretical_loss": 3.6358630092452504, + "tokens_seen": 1039335424 + }, + { + "epoch": 0.31, + "learning_rate": 0.0003460180541624875, + "loss": 2.5954, + "theoretical_loss": 3.6358413472824838, + "tokens_seen": 1039400960 + }, + { + "epoch": 0.31, + "learning_rate": 0.00034600802407221663, + "loss": 2.5512, + "theoretical_loss": 3.635819687067901, + "tokens_seen": 1039466496 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034599799398194587, + "loss": 2.6184, + "theoretical_loss": 3.6357980286012506, + "tokens_seen": 1039532032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034598796389167505, + "loss": 2.7418, + "theoretical_loss": 3.6357763718822813, + "tokens_seen": 1039597568 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034597793380140423, + "loss": 2.6507, + "theoretical_loss": 3.635754716910742, + "tokens_seen": 1039663104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003459679037111334, + "loss": 2.3988, + "theoretical_loss": 3.6357330636863816, + "tokens_seen": 1039728640 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034595787362086264, + "loss": 2.6205, + "theoretical_loss": 3.6357114122089493, + "tokens_seen": 1039794176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034594784353059177, + "loss": 2.858, + "theoretical_loss": 3.6356897624781936, + "tokens_seen": 1039859712 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345937813440321, + "loss": 2.6748, + "theoretical_loss": 3.635668114493863, + "tokens_seen": 1039925248 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034592778335005013, + "loss": 2.5437, + "theoretical_loss": 3.635646468255708, + "tokens_seen": 1039990784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034591775325977937, + "loss": 2.4441, + "theoretical_loss": 3.635624823763477, + "tokens_seen": 1040056320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034590772316950855, + "loss": 2.562, + "theoretical_loss": 3.6356031810169185, + "tokens_seen": 1040121856 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034589769307923773, + "loss": 2.7612, + "theoretical_loss": 3.635581540015783, + "tokens_seen": 1040187392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458876629889669, + "loss": 2.6531, + "theoretical_loss": 3.635559900759819, + "tokens_seen": 1040252928 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458776328986961, + "loss": 2.8936, + "theoretical_loss": 3.635538263248776, + "tokens_seen": 1040318464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 580938, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.539257049560547, + "objective/train/theoretical_loss": 3.635516627482403, + "objective/train/tokens_used": 1060844000, + "theoretical_loss": 3.635516627482403, + "tokens_seen": 1040384000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458676028084253, + "loss": 2.5525, + "theoretical_loss": 3.635516627482403, + "tokens_seen": 1040384000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458575727181545, + "loss": 2.6589, + "theoretical_loss": 3.6354949934604504, + "tokens_seen": 1040449536 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034584754262788364, + "loss": 2.7115, + "theoretical_loss": 3.6354733611826666, + "tokens_seen": 1040515072 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034583751253761287, + "loss": 2.6772, + "theoretical_loss": 3.6354517306488017, + "tokens_seen": 1040580608 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345827482447342, + "loss": 2.6409, + "theoretical_loss": 3.6354301018586055, + "tokens_seen": 1040646144 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034581745235707123, + "loss": 2.9847, + "theoretical_loss": 3.635408474811827, + "tokens_seen": 1040711680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003458074222668004, + "loss": 2.5639, + "theoretical_loss": 3.635386849508217, + "tokens_seen": 1040777216 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457973921765296, + "loss": 2.6658, + "theoretical_loss": 3.635365225947524, + "tokens_seen": 1040842752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457873620862588, + "loss": 2.5532, + "theoretical_loss": 3.635343604129498, + "tokens_seen": 1040908288 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345777331995988, + "loss": 2.7893, + "theoretical_loss": 3.63532198405389, + "tokens_seen": 1040973824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034576730190571714, + "loss": 2.6635, + "theoretical_loss": 3.6353003657204486, + "tokens_seen": 1041039360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457572718154464, + "loss": 2.5693, + "theoretical_loss": 3.635278749128924, + "tokens_seen": 1041104896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457472417251755, + "loss": 2.6342, + "theoretical_loss": 3.635257134279067, + "tokens_seen": 1041170432 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034573721163490474, + "loss": 2.6137, + "theoretical_loss": 3.6352355211706264, + "tokens_seen": 1041235968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457271815446339, + "loss": 2.7036, + "theoretical_loss": 3.635213909803354, + "tokens_seen": 1041301504 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457171514543631, + "loss": 2.712, + "theoretical_loss": 3.635192300176998, + "tokens_seen": 1041367040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003457071213640923, + "loss": 2.7841, + "theoretical_loss": 3.63517069229131, + "tokens_seen": 1041432576 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034569709127382146, + "loss": 2.7111, + "theoretical_loss": 3.63514908614604, + "tokens_seen": 1041498112 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034568706118355064, + "loss": 2.717, + "theoretical_loss": 3.6351274817409376, + "tokens_seen": 1041563648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456770310932799, + "loss": 2.4966, + "theoretical_loss": 3.6351058790757538, + "tokens_seen": 1041629184 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345667001003009, + "loss": 2.7355, + "theoretical_loss": 3.6350842781502393, + "tokens_seen": 1041694720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034565697091273824, + "loss": 2.691, + "theoretical_loss": 3.635062678964144, + "tokens_seen": 1041760256 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034564694082246737, + "loss": 2.6776, + "theoretical_loss": 3.635041081517219, + "tokens_seen": 1041825792 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456369107321966, + "loss": 2.5323, + "theoretical_loss": 3.635019485809214, + "tokens_seen": 1041891328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003456268806419258, + "loss": 2.701, + "theoretical_loss": 3.6349978918398804, + "tokens_seen": 1041956864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 581557, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8283419609069824, + "objective/train/theoretical_loss": 3.6349762996089683, + "objective/train/tokens_used": 1062482400, + "theoretical_loss": 3.6349762996089683, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034561685055165496, + "loss": 2.6366, + "theoretical_loss": 3.6349762996089683, + "tokens_seen": 1042022400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034560682046138415, + "loss": 2.694, + "theoretical_loss": 3.634954709116229, + "tokens_seen": 1042087936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455967903711134, + "loss": 2.6484, + "theoretical_loss": 3.6349331203614126, + "tokens_seen": 1042153472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455867602808425, + "loss": 2.6699, + "theoretical_loss": 3.6349115333442708, + "tokens_seen": 1042219008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034557673019057174, + "loss": 2.8731, + "theoretical_loss": 3.6348899480645542, + "tokens_seen": 1042284544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034556670010030087, + "loss": 2.5907, + "theoretical_loss": 3.634868364522013, + "tokens_seen": 1042350080 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455566700100301, + "loss": 2.7887, + "theoretical_loss": 3.634846782716399, + "tokens_seen": 1042415616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003455466399197593, + "loss": 2.58, + "theoretical_loss": 3.634825202647463, + "tokens_seen": 1042481152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034553660982948847, + "loss": 2.5548, + "theoretical_loss": 3.634803624314956, + "tokens_seen": 1042546688 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034552657973921765, + "loss": 2.6047, + "theoretical_loss": 3.634782047718629, + "tokens_seen": 1042612224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034551654964894683, + "loss": 2.6368, + "theoretical_loss": 3.6347604728582334, + "tokens_seen": 1042677760 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345506519558676, + "loss": 2.752, + "theoretical_loss": 3.6347388997335206, + "tokens_seen": 1042743296 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034549648946840525, + "loss": 2.7275, + "theoretical_loss": 3.6347173283442418, + "tokens_seen": 1042808832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454864593781344, + "loss": 2.6721, + "theoretical_loss": 3.634695758690148, + "tokens_seen": 1042874368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454764292878636, + "loss": 2.7311, + "theoretical_loss": 3.6346741907709914, + "tokens_seen": 1042939904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034546639919759274, + "loss": 2.7933, + "theoretical_loss": 3.634652624586522, + "tokens_seen": 1043005440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034545636910732197, + "loss": 2.5669, + "theoretical_loss": 3.634631060136493, + "tokens_seen": 1043070976 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034544633901705115, + "loss": 2.6247, + "theoretical_loss": 3.6346094974206546, + "tokens_seen": 1043136512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034543630892678033, + "loss": 2.7361, + "theoretical_loss": 3.634587936438759, + "tokens_seen": 1043202048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454262788365095, + "loss": 2.385, + "theoretical_loss": 3.6345663771905583, + "tokens_seen": 1043267584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034541624874623875, + "loss": 2.6379, + "theoretical_loss": 3.6345448196758032, + "tokens_seen": 1043333120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003454062186559679, + "loss": 2.7274, + "theoretical_loss": 3.634523263894246, + "tokens_seen": 1043398656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453961885656971, + "loss": 2.6841, + "theoretical_loss": 3.6345017098456385, + "tokens_seen": 1043464192 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034538615847542624, + "loss": 2.5927, + "theoretical_loss": 3.6344801575297323, + "tokens_seen": 1043529728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453761283851555, + "loss": 2.718, + "theoretical_loss": 3.63445860694628, + "tokens_seen": 1043595264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 582812, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.661972761154175, + "objective/train/theoretical_loss": 3.634437058095033, + "objective/train/tokens_used": 1064120800, + "theoretical_loss": 3.634437058095033, + "tokens_seen": 1043660800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034536609829488466, + "loss": 2.6098, + "theoretical_loss": 3.634437058095033, + "tokens_seen": 1043660800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034535606820461384, + "loss": 2.8238, + "theoretical_loss": 3.6344155109757432, + "tokens_seen": 1043726336 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345346038114343, + "loss": 2.8692, + "theoretical_loss": 3.6343939655881625, + "tokens_seen": 1043791872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453360080240722, + "loss": 2.5667, + "theoretical_loss": 3.634372421932044, + "tokens_seen": 1043857408 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034532597793380143, + "loss": 2.9155, + "theoretical_loss": 3.6343508800071387, + "tokens_seen": 1043922944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453159478435306, + "loss": 2.7324, + "theoretical_loss": 3.634329339813199, + "tokens_seen": 1043988480 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003453059177532598, + "loss": 2.6672, + "theoretical_loss": 3.634307801349978, + "tokens_seen": 1044054016 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345295887662989, + "loss": 2.6618, + "theoretical_loss": 3.6342862646172276, + "tokens_seen": 1044119552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452858575727182, + "loss": 2.7461, + "theoretical_loss": 3.6342647296147, + "tokens_seen": 1044185088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034527582748244734, + "loss": 2.7409, + "theoretical_loss": 3.6342431963421475, + "tokens_seen": 1044250624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452657973921766, + "loss": 2.7384, + "theoretical_loss": 3.634221664799323, + "tokens_seen": 1044316160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452557673019057, + "loss": 2.7435, + "theoretical_loss": 3.6342001349859783, + "tokens_seen": 1044381696 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034524573721163494, + "loss": 2.7216, + "theoretical_loss": 3.634178606901867, + "tokens_seen": 1044447232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452357071213641, + "loss": 2.6133, + "theoretical_loss": 3.6341570805467405, + "tokens_seen": 1044512768 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452256770310933, + "loss": 2.6797, + "theoretical_loss": 3.6341355559203525, + "tokens_seen": 1044578304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003452156469408225, + "loss": 2.7254, + "theoretical_loss": 3.6341140330224553, + "tokens_seen": 1044643840 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034520561685055166, + "loss": 2.627, + "theoretical_loss": 3.634092511852802, + "tokens_seen": 1044709376 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034519558676028084, + "loss": 2.5409, + "theoretical_loss": 3.6340709924111447, + "tokens_seen": 1044774912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451855566700101, + "loss": 2.5336, + "theoretical_loss": 3.634049474697237, + "tokens_seen": 1044840448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451755265797392, + "loss": 2.7567, + "theoretical_loss": 3.6340279587108313, + "tokens_seen": 1044905984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034516549648946844, + "loss": 2.6346, + "theoretical_loss": 3.634006444451681, + "tokens_seen": 1044971520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034515546639919757, + "loss": 2.5416, + "theoretical_loss": 3.6339849319195388, + "tokens_seen": 1045037056 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451454363089268, + "loss": 2.5917, + "theoretical_loss": 3.6339634211141583, + "tokens_seen": 1045102592 + }, + { + "epoch": 0.32, + "learning_rate": 0.000345135406218656, + "loss": 2.7407, + "theoretical_loss": 3.633941912035292, + "tokens_seen": 1045168128 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034512537612838517, + "loss": 2.4952, + "theoretical_loss": 3.633920404682693, + "tokens_seen": 1045233664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 583395, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6361796855926514, + "objective/train/theoretical_loss": 3.633898899056115, + "objective/train/tokens_used": 1065759200, + "theoretical_loss": 3.633898899056115, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034511534603811435, + "loss": 2.7576, + "theoretical_loss": 3.633898899056115, + "tokens_seen": 1045299200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003451053159478436, + "loss": 2.5862, + "theoretical_loss": 3.633877395155311, + "tokens_seen": 1045364736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450952858575727, + "loss": 2.631, + "theoretical_loss": 3.6338558929800353, + "tokens_seen": 1045430272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034508525576730194, + "loss": 2.8207, + "theoretical_loss": 3.63383439253004, + "tokens_seen": 1045495808 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034507522567703107, + "loss": 2.5697, + "theoretical_loss": 3.6338128938050787, + "tokens_seen": 1045561344 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450651955867603, + "loss": 2.6425, + "theoretical_loss": 3.633791396804905, + "tokens_seen": 1045626880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450551654964895, + "loss": 2.7739, + "theoretical_loss": 3.6337699015292735, + "tokens_seen": 1045692416 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034504513540621867, + "loss": 2.3681, + "theoretical_loss": 3.633748407977936, + "tokens_seen": 1045757952 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034503510531594785, + "loss": 2.8102, + "theoretical_loss": 3.6337269161506476, + "tokens_seen": 1045823488 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034502507522567703, + "loss": 2.4205, + "theoretical_loss": 3.633705426047161, + "tokens_seen": 1045889024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003450150451354062, + "loss": 2.5667, + "theoretical_loss": 3.6336839376672305, + "tokens_seen": 1045954560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034500501504513545, + "loss": 2.4626, + "theoretical_loss": 3.63366245101061, + "tokens_seen": 1046020096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449949849548646, + "loss": 2.503, + "theoretical_loss": 3.633640966077053, + "tokens_seen": 1046085632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449849548645938, + "loss": 2.7242, + "theoretical_loss": 3.633619482866313, + "tokens_seen": 1046151168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034497492477432294, + "loss": 2.7701, + "theoretical_loss": 3.633598001378145, + "tokens_seen": 1046216704 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034496489468405217, + "loss": 2.6704, + "theoretical_loss": 3.633576521612302, + "tokens_seen": 1046282240 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034495486459378135, + "loss": 2.5448, + "theoretical_loss": 3.6335550435685384, + "tokens_seen": 1046347776 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034494483450351053, + "loss": 2.559, + "theoretical_loss": 3.6335335672466087, + "tokens_seen": 1046413312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449348044132397, + "loss": 2.853, + "theoretical_loss": 3.6335120926462663, + "tokens_seen": 1046478848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034492477432296895, + "loss": 2.4466, + "theoretical_loss": 3.633490619767266, + "tokens_seen": 1046544384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449147442326981, + "loss": 2.609, + "theoretical_loss": 3.6334691486093615, + "tokens_seen": 1046609920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003449047141424273, + "loss": 2.5306, + "theoretical_loss": 3.633447679172308, + "tokens_seen": 1046675456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034489468405215644, + "loss": 2.4962, + "theoretical_loss": 3.6334262114558586, + "tokens_seen": 1046740992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448846539618857, + "loss": 2.8402, + "theoretical_loss": 3.6334047454597687, + "tokens_seen": 1046806528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034487462387161486, + "loss": 2.4984, + "theoretical_loss": 3.633383281183792, + "tokens_seen": 1046872064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 584517, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.44215989112854, + "objective/train/theoretical_loss": 3.6333618186276837, + "objective/train/tokens_used": 1067397600, + "theoretical_loss": 3.6333618186276837, + "tokens_seen": 1046937600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034486459378134404, + "loss": 2.5829, + "theoretical_loss": 3.6333618186276837, + "tokens_seen": 1046937600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448545636910732, + "loss": 2.6031, + "theoretical_loss": 3.633340357791198, + "tokens_seen": 1047003136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448445336008024, + "loss": 2.5413, + "theoretical_loss": 3.6333188986740894, + "tokens_seen": 1047068672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448345035105316, + "loss": 2.5823, + "theoretical_loss": 3.6332974412761128, + "tokens_seen": 1047134208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448244734202608, + "loss": 2.5924, + "theoretical_loss": 3.6332759855970225, + "tokens_seen": 1047199744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034481444332998994, + "loss": 2.7455, + "theoretical_loss": 3.633254531636574, + "tokens_seen": 1047265280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003448044132397192, + "loss": 2.4347, + "theoretical_loss": 3.633233079394521, + "tokens_seen": 1047330816 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034479438314944836, + "loss": 2.6795, + "theoretical_loss": 3.633211628870619, + "tokens_seen": 1047396352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034478435305917754, + "loss": 2.5025, + "theoretical_loss": 3.6331901800646236, + "tokens_seen": 1047461888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447743229689067, + "loss": 2.679, + "theoretical_loss": 3.633168732976288, + "tokens_seen": 1047527424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447642928786359, + "loss": 2.6558, + "theoretical_loss": 3.633147287605369, + "tokens_seen": 1047592960 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447542627883651, + "loss": 2.6364, + "theoretical_loss": 3.6331258439516203, + "tokens_seen": 1047658496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447442326980943, + "loss": 2.4161, + "theoretical_loss": 3.633104402014798, + "tokens_seen": 1047724032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034473420260782345, + "loss": 2.5527, + "theoretical_loss": 3.633082961794656, + "tokens_seen": 1047789568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447241725175527, + "loss": 2.5928, + "theoretical_loss": 3.6330615232909507, + "tokens_seen": 1047855104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003447141424272818, + "loss": 2.625, + "theoretical_loss": 3.6330400865034367, + "tokens_seen": 1047920640 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034470411233701104, + "loss": 2.6341, + "theoretical_loss": 3.63301865143187, + "tokens_seen": 1047986176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446940822467402, + "loss": 2.5154, + "theoretical_loss": 3.6329972180760057, + "tokens_seen": 1048051712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446840521564694, + "loss": 2.5054, + "theoretical_loss": 3.632975786435598, + "tokens_seen": 1048117248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446740220661986, + "loss": 2.7753, + "theoretical_loss": 3.632954356510404, + "tokens_seen": 1048182784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034466399197592777, + "loss": 2.8493, + "theoretical_loss": 3.6329329283001783, + "tokens_seen": 1048248320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034465396188565695, + "loss": 2.6918, + "theoretical_loss": 3.632911501804677, + "tokens_seen": 1048313856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446439317953862, + "loss": 2.6121, + "theoretical_loss": 3.632890077023655, + "tokens_seen": 1048379392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446339017051153, + "loss": 2.5529, + "theoretical_loss": 3.6328686539568684, + "tokens_seen": 1048444928 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034462387161484455, + "loss": 2.4928, + "theoretical_loss": 3.6328472326040737, + "tokens_seen": 1048510464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 585002, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 1.9021494388580322, + "objective/train/theoretical_loss": 3.6328258129650246, + "objective/train/tokens_used": 1069036000, + "theoretical_loss": 3.6328258129650246, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034461384152457373, + "loss": 2.5486, + "theoretical_loss": 3.6328258129650246, + "tokens_seen": 1048576000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003446038114343029, + "loss": 2.7026, + "theoretical_loss": 3.6328043950394786, + "tokens_seen": 1048641536 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034459378134403214, + "loss": 2.4971, + "theoretical_loss": 3.632782978827191, + "tokens_seen": 1048707072 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034458375125376127, + "loss": 2.5622, + "theoretical_loss": 3.6327615643279176, + "tokens_seen": 1048772608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445737211634905, + "loss": 2.7683, + "theoretical_loss": 3.6327401515414146, + "tokens_seen": 1048838144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445636910732197, + "loss": 2.612, + "theoretical_loss": 3.632718740467438, + "tokens_seen": 1048903680 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034455366098294887, + "loss": 2.8323, + "theoretical_loss": 3.6326973311057436, + "tokens_seen": 1048969216 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034454363089267805, + "loss": 2.6183, + "theoretical_loss": 3.6326759234560875, + "tokens_seen": 1049034752 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034453360080240723, + "loss": 2.6601, + "theoretical_loss": 3.632654517518226, + "tokens_seen": 1049100288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445235707121364, + "loss": 2.8211, + "theoretical_loss": 3.6326331132919156, + "tokens_seen": 1049165824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034451354062186565, + "loss": 2.6244, + "theoretical_loss": 3.632611710776912, + "tokens_seen": 1049231360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003445035105315948, + "loss": 2.4925, + "theoretical_loss": 3.632590309972972, + "tokens_seen": 1049296896 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344493480441324, + "loss": 2.9594, + "theoretical_loss": 3.6325689108798516, + "tokens_seen": 1049362432 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034448345035105314, + "loss": 2.7034, + "theoretical_loss": 3.6325475134973066, + "tokens_seen": 1049427968 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034447342026078237, + "loss": 2.5489, + "theoretical_loss": 3.632526117825095, + "tokens_seen": 1049493504 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034446339017051155, + "loss": 2.6613, + "theoretical_loss": 3.632504723862972, + "tokens_seen": 1049559040 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034445336008024073, + "loss": 2.3941, + "theoretical_loss": 3.632483331610695, + "tokens_seen": 1049624576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444433299899699, + "loss": 2.6026, + "theoretical_loss": 3.63246194106802, + "tokens_seen": 1049690112 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034443329989969915, + "loss": 2.8111, + "theoretical_loss": 3.632440552234704, + "tokens_seen": 1049755648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444232698094283, + "loss": 2.7581, + "theoretical_loss": 3.6324191651105036, + "tokens_seen": 1049821184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003444132397191575, + "loss": 2.5065, + "theoretical_loss": 3.632397779695175, + "tokens_seen": 1049886720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034440320962888664, + "loss": 2.4674, + "theoretical_loss": 3.632376395988476, + "tokens_seen": 1049952256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443931795386159, + "loss": 2.3677, + "theoretical_loss": 3.632355013990163, + "tokens_seen": 1050017792 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034438314944834506, + "loss": 2.6539, + "theoretical_loss": 3.632333633699992, + "tokens_seen": 1050083328 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034437311935807424, + "loss": 2.3852, + "theoretical_loss": 3.6323122551177214, + "tokens_seen": 1050148864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 586227, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.932581663131714, + "objective/train/theoretical_loss": 3.632290878243108, + "objective/train/tokens_used": 1070674400, + "theoretical_loss": 3.632290878243108, + "tokens_seen": 1050214400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443630892678034, + "loss": 2.6061, + "theoretical_loss": 3.632290878243108, + "tokens_seen": 1050214400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443530591775326, + "loss": 2.6796, + "theoretical_loss": 3.632269503075908, + "tokens_seen": 1050279936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443430290872618, + "loss": 2.7109, + "theoretical_loss": 3.6322481296158786, + "tokens_seen": 1050345472 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344332998996991, + "loss": 2.5937, + "theoretical_loss": 3.632226757862778, + "tokens_seen": 1050411008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034432296890672014, + "loss": 2.7738, + "theoretical_loss": 3.632205387816362, + "tokens_seen": 1050476544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003443129388164494, + "loss": 2.7724, + "theoretical_loss": 3.632184019476389, + "tokens_seen": 1050542080 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034430290872617856, + "loss": 2.635, + "theoretical_loss": 3.6321626528426156, + "tokens_seen": 1050607616 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034429287863590774, + "loss": 2.5516, + "theoretical_loss": 3.6321412879147994, + "tokens_seen": 1050673152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442828485456369, + "loss": 2.7096, + "theoretical_loss": 3.632119924692698, + "tokens_seen": 1050738688 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442728184553661, + "loss": 2.5264, + "theoretical_loss": 3.632098563176068, + "tokens_seen": 1050804224 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442627883650953, + "loss": 2.4598, + "theoretical_loss": 3.632077203364668, + "tokens_seen": 1050869760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442527582748245, + "loss": 2.4605, + "theoretical_loss": 3.632055845258255, + "tokens_seen": 1050935296 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034424272818455365, + "loss": 2.6781, + "theoretical_loss": 3.6320344888565863, + "tokens_seen": 1051000832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442326980942829, + "loss": 2.6028, + "theoretical_loss": 3.6320131341594206, + "tokens_seen": 1051066368 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344222668004012, + "loss": 2.5341, + "theoretical_loss": 3.6319917811665143, + "tokens_seen": 1051131904 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034421263791374124, + "loss": 2.5869, + "theoretical_loss": 3.631970429877626, + "tokens_seen": 1051197440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003442026078234704, + "loss": 2.7181, + "theoretical_loss": 3.6319490802925136, + "tokens_seen": 1051262976 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441925777331996, + "loss": 2.8044, + "theoretical_loss": 3.631927732410934, + "tokens_seen": 1051328512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441825476429288, + "loss": 2.6143, + "theoretical_loss": 3.6319063862326457, + "tokens_seen": 1051394048 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034417251755265797, + "loss": 2.8513, + "theoretical_loss": 3.631885041757407, + "tokens_seen": 1051459584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034416248746238715, + "loss": 2.5423, + "theoretical_loss": 3.631863698984975, + "tokens_seen": 1051525120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441524573721164, + "loss": 2.5051, + "theoretical_loss": 3.631842357915108, + "tokens_seen": 1051590656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441424272818455, + "loss": 2.4431, + "theoretical_loss": 3.631821018547565, + "tokens_seen": 1051656192 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034413239719157475, + "loss": 2.6653, + "theoretical_loss": 3.6317996808821027, + "tokens_seen": 1051721728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034412236710130393, + "loss": 2.5794, + "theoretical_loss": 3.631778344918481, + "tokens_seen": 1051787264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 587166, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6821694374084473, + "objective/train/theoretical_loss": 3.6317570106564565, + "objective/train/tokens_used": 1072312800, + "theoretical_loss": 3.6317570106564565, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441123370110331, + "loss": 2.6543, + "theoretical_loss": 3.6317570106564565, + "tokens_seen": 1051852800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003441023069207623, + "loss": 2.6049, + "theoretical_loss": 3.6317356780957883, + "tokens_seen": 1051918336 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034409227683049147, + "loss": 2.2439, + "theoretical_loss": 3.6317143472362345, + "tokens_seen": 1051983872 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034408224674022065, + "loss": 2.7155, + "theoretical_loss": 3.631693018077554, + "tokens_seen": 1052049408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440722166499499, + "loss": 2.5276, + "theoretical_loss": 3.6316716906195046, + "tokens_seen": 1052114944 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344062186559679, + "loss": 2.512, + "theoretical_loss": 3.6316503648618452, + "tokens_seen": 1052180480 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034405215646940825, + "loss": 2.7207, + "theoretical_loss": 3.6316290408043335, + "tokens_seen": 1052246016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440421263791374, + "loss": 2.6449, + "theoretical_loss": 3.6316077184467295, + "tokens_seen": 1052311552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440320962888666, + "loss": 2.6526, + "theoretical_loss": 3.6315863977887908, + "tokens_seen": 1052377088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003440220661985958, + "loss": 2.6497, + "theoretical_loss": 3.6315650788302762, + "tokens_seen": 1052442624 + }, + { + "epoch": 0.32, + "learning_rate": 0.000344012036108325, + "loss": 2.7508, + "theoretical_loss": 3.6315437615709447, + "tokens_seen": 1052508160 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034400200601805416, + "loss": 2.8709, + "theoretical_loss": 3.631522446010555, + "tokens_seen": 1052573696 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034399197592778334, + "loss": 2.565, + "theoretical_loss": 3.6315011321488657, + "tokens_seen": 1052639232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439819458375125, + "loss": 2.6508, + "theoretical_loss": 3.631479819985636, + "tokens_seen": 1052704768 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034397191574724175, + "loss": 2.7176, + "theoretical_loss": 3.6314585095206247, + "tokens_seen": 1052770304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439618856569709, + "loss": 2.7527, + "theoretical_loss": 3.6314372007535907, + "tokens_seen": 1052835840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439518555667001, + "loss": 2.631, + "theoretical_loss": 3.6314158936842933, + "tokens_seen": 1052901376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439418254764293, + "loss": 2.7249, + "theoretical_loss": 3.6313945883124914, + "tokens_seen": 1052966912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003439317953861585, + "loss": 2.6789, + "theoretical_loss": 3.6313732846379434, + "tokens_seen": 1053032448 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034392176529588766, + "loss": 2.8109, + "theoretical_loss": 3.63135198266041, + "tokens_seen": 1053097984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034391173520561684, + "loss": 2.4002, + "theoretical_loss": 3.631330682379649, + "tokens_seen": 1053163520 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343901705115346, + "loss": 2.6808, + "theoretical_loss": 3.631309383795421, + "tokens_seen": 1053229056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034389167502507526, + "loss": 2.5586, + "theoretical_loss": 3.631288086907484, + "tokens_seen": 1053294592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438816449348044, + "loss": 2.6458, + "theoretical_loss": 3.631266791715598, + "tokens_seen": 1053360128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438716148445336, + "loss": 2.9422, + "theoretical_loss": 3.6312454982195224, + "tokens_seen": 1053425664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 587595, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.472946882247925, + "objective/train/theoretical_loss": 3.631224206419017, + "objective/train/tokens_used": 1073951200, + "theoretical_loss": 3.631224206419017, + "tokens_seen": 1053491200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034386158475426275, + "loss": 2.5474, + "theoretical_loss": 3.631224206419017, + "tokens_seen": 1053491200 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343851554663992, + "loss": 2.5543, + "theoretical_loss": 3.631202916313841, + "tokens_seen": 1053556736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438415245737212, + "loss": 2.6514, + "theoretical_loss": 3.6311816279037537, + "tokens_seen": 1053622272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034383149448345034, + "loss": 2.71, + "theoretical_loss": 3.6311603411885147, + "tokens_seen": 1053687808 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003438214643931796, + "loss": 2.6042, + "theoretical_loss": 3.631139056167884, + "tokens_seen": 1053753344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034381143430290876, + "loss": 2.5687, + "theoretical_loss": 3.6311177728416215, + "tokens_seen": 1053818880 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034380140421263794, + "loss": 2.7691, + "theoretical_loss": 3.6310964912094867, + "tokens_seen": 1053884416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437913741223671, + "loss": 2.7997, + "theoretical_loss": 3.6310752112712392, + "tokens_seen": 1053949952 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437813440320963, + "loss": 2.4598, + "theoretical_loss": 3.631053933026639, + "tokens_seen": 1054015488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437713139418255, + "loss": 2.7018, + "theoretical_loss": 3.6310326564754467, + "tokens_seen": 1054081024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437612838515547, + "loss": 2.596, + "theoretical_loss": 3.631011381617421, + "tokens_seen": 1054146560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034375125376128385, + "loss": 2.9412, + "theoretical_loss": 3.630990108452323, + "tokens_seen": 1054212096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437412236710131, + "loss": 2.6816, + "theoretical_loss": 3.630968836979912, + "tokens_seen": 1054277632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437311935807422, + "loss": 2.6108, + "theoretical_loss": 3.6309475671999483, + "tokens_seen": 1054343168 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034372116349047144, + "loss": 2.6602, + "theoretical_loss": 3.6309262991121924, + "tokens_seen": 1054408704 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437111334002006, + "loss": 2.4562, + "theoretical_loss": 3.630905032716404, + "tokens_seen": 1054474240 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003437011033099298, + "loss": 2.6896, + "theoretical_loss": 3.630883768012344, + "tokens_seen": 1054539776 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343691073219659, + "loss": 2.8709, + "theoretical_loss": 3.6308625049997723, + "tokens_seen": 1054605312 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034368104312938817, + "loss": 2.5781, + "theoretical_loss": 3.630841243678449, + "tokens_seen": 1054670848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034367101303911735, + "loss": 2.7216, + "theoretical_loss": 3.630819984048135, + "tokens_seen": 1054736384 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436609829488466, + "loss": 2.6508, + "theoretical_loss": 3.6307987261085906, + "tokens_seen": 1054801920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436509528585757, + "loss": 2.6724, + "theoretical_loss": 3.630777469859576, + "tokens_seen": 1054867456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034364092276830495, + "loss": 2.6541, + "theoretical_loss": 3.6307562153008517, + "tokens_seen": 1054932992 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034363089267803413, + "loss": 2.901, + "theoretical_loss": 3.6307349624321787, + "tokens_seen": 1054998528 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436208625877633, + "loss": 2.5115, + "theoretical_loss": 3.630713711253318, + "tokens_seen": 1055064064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 588937, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4390525817871094, + "objective/train/theoretical_loss": 3.6306924617640295, + "objective/train/tokens_used": 1075589600, + "theoretical_loss": 3.6306924617640295, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003436108324974925, + "loss": 2.5418, + "theoretical_loss": 3.6306924617640295, + "tokens_seen": 1055129600 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034360080240722167, + "loss": 2.7519, + "theoretical_loss": 3.6306712139640744, + "tokens_seen": 1055195136 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034359077231695085, + "loss": 2.6733, + "theoretical_loss": 3.6306499678532127, + "tokens_seen": 1055260672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435807422266801, + "loss": 2.5918, + "theoretical_loss": 3.6306287234312062, + "tokens_seen": 1055326208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435707121364092, + "loss": 2.5974, + "theoretical_loss": 3.630607480697816, + "tokens_seen": 1055391744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034356068204613845, + "loss": 2.6814, + "theoretical_loss": 3.6305862396528017, + "tokens_seen": 1055457280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435506519558676, + "loss": 2.6467, + "theoretical_loss": 3.6305650002959258, + "tokens_seen": 1055522816 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435406218655968, + "loss": 2.5512, + "theoretical_loss": 3.6305437626269486, + "tokens_seen": 1055588352 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343530591775326, + "loss": 2.6324, + "theoretical_loss": 3.6305225266456302, + "tokens_seen": 1055653888 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003435205616850552, + "loss": 2.6879, + "theoretical_loss": 3.6305012923517337, + "tokens_seen": 1055719424 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034351053159478436, + "loss": 2.3758, + "theoretical_loss": 3.630480059745019, + "tokens_seen": 1055784960 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034350050150451354, + "loss": 2.7239, + "theoretical_loss": 3.6304588288252475, + "tokens_seen": 1055850496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434904714142427, + "loss": 2.6559, + "theoretical_loss": 3.630437599592181, + "tokens_seen": 1055916032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034348044132397195, + "loss": 2.4738, + "theoretical_loss": 3.6304163720455795, + "tokens_seen": 1055981568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434704112337011, + "loss": 2.7831, + "theoretical_loss": 3.630395146185206, + "tokens_seen": 1056047104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434603811434303, + "loss": 2.6243, + "theoretical_loss": 3.630373922010821, + "tokens_seen": 1056112640 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434503510531595, + "loss": 2.6946, + "theoretical_loss": 3.6303526995221866, + "tokens_seen": 1056178176 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434403209628887, + "loss": 2.6034, + "theoretical_loss": 3.6303314787190635, + "tokens_seen": 1056243712 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034343029087261786, + "loss": 2.5507, + "theoretical_loss": 3.630310259601214, + "tokens_seen": 1056309248 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034342026078234704, + "loss": 2.5951, + "theoretical_loss": 3.630289042168399, + "tokens_seen": 1056374784 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003434102306920762, + "loss": 2.7226, + "theoretical_loss": 3.630267826420381, + "tokens_seen": 1056440320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034340020060180546, + "loss": 2.7196, + "theoretical_loss": 3.630246612356921, + "tokens_seen": 1056505856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433901705115346, + "loss": 2.6928, + "theoretical_loss": 3.630225399977781, + "tokens_seen": 1056571392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433801404212638, + "loss": 2.5232, + "theoretical_loss": 3.630204189282723, + "tokens_seen": 1056636928 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034337011033099295, + "loss": 2.7453, + "theoretical_loss": 3.6301829802715084, + "tokens_seen": 1056702464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 589659, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.753281354904175, + "objective/train/theoretical_loss": 3.6301617729438993, + "objective/train/tokens_used": 1077228000, + "theoretical_loss": 3.6301617729438993, + "tokens_seen": 1056768000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433600802407222, + "loss": 2.8091, + "theoretical_loss": 3.6301617729438993, + "tokens_seen": 1056768000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034335005015045136, + "loss": 2.6981, + "theoretical_loss": 3.6301405672996583, + "tokens_seen": 1056833536 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034334002006018054, + "loss": 2.6649, + "theoretical_loss": 3.630119363338547, + "tokens_seen": 1056899072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433299899699097, + "loss": 2.6571, + "theoretical_loss": 3.6300981610603267, + "tokens_seen": 1056964608 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433199598796389, + "loss": 2.6702, + "theoretical_loss": 3.6300769604647605, + "tokens_seen": 1057030144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003433099297893681, + "loss": 2.8513, + "theoretical_loss": 3.63005576155161, + "tokens_seen": 1057095680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432998996990973, + "loss": 2.744, + "theoretical_loss": 3.6300345643206375, + "tokens_seen": 1057161216 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034328986960882645, + "loss": 2.4055, + "theoretical_loss": 3.6300133687716056, + "tokens_seen": 1057226752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432798395185557, + "loss": 2.6993, + "theoretical_loss": 3.6299921749042765, + "tokens_seen": 1057292288 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034326980942828487, + "loss": 2.3819, + "theoretical_loss": 3.6299709827184117, + "tokens_seen": 1057357824 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034325977933801405, + "loss": 2.8359, + "theoretical_loss": 3.6299497922137745, + "tokens_seen": 1057423360 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034324974924774323, + "loss": 2.6268, + "theoretical_loss": 3.6299286033901272, + "tokens_seen": 1057488896 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432397191574724, + "loss": 2.7105, + "theoretical_loss": 3.6299074162472325, + "tokens_seen": 1057554432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432296890672016, + "loss": 2.4748, + "theoretical_loss": 3.6298862307848516, + "tokens_seen": 1057619968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003432196589769308, + "loss": 2.8688, + "theoretical_loss": 3.629865047002749, + "tokens_seen": 1057685504 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034320962888665995, + "loss": 2.6851, + "theoretical_loss": 3.6298438649006863, + "tokens_seen": 1057751040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431995987963892, + "loss": 2.7079, + "theoretical_loss": 3.629822684478426, + "tokens_seen": 1057816576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431895687061183, + "loss": 2.8602, + "theoretical_loss": 3.629801505735731, + "tokens_seen": 1057882112 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034317953861584755, + "loss": 2.5481, + "theoretical_loss": 3.629780328672364, + "tokens_seen": 1057947648 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034316950852557673, + "loss": 2.8164, + "theoretical_loss": 3.629759153288089, + "tokens_seen": 1058013184 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431594784353059, + "loss": 2.6221, + "theoretical_loss": 3.6297379795826674, + "tokens_seen": 1058078720 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431494483450351, + "loss": 2.563, + "theoretical_loss": 3.629716807555863, + "tokens_seen": 1058144256 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034313941825476433, + "loss": 2.6197, + "theoretical_loss": 3.629695637207438, + "tokens_seen": 1058209792 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034312938816449345, + "loss": 2.6229, + "theoretical_loss": 3.6296744685371554, + "tokens_seen": 1058275328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431193580742227, + "loss": 2.4343, + "theoretical_loss": 3.629653301544779, + "tokens_seen": 1058340864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 591033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.53261661529541, + "objective/train/theoretical_loss": 3.6296321362300716, + "objective/train/tokens_used": 1078866400, + "theoretical_loss": 3.6296321362300716, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003431093279839518, + "loss": 2.5809, + "theoretical_loss": 3.6296321362300716, + "tokens_seen": 1058406400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034309929789368105, + "loss": 2.5506, + "theoretical_loss": 3.629610972592797, + "tokens_seen": 1058471936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430892678034103, + "loss": 2.8012, + "theoretical_loss": 3.6295898106327167, + "tokens_seen": 1058537472 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430792377131394, + "loss": 2.7142, + "theoretical_loss": 3.629568650349596, + "tokens_seen": 1058603008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034306920762286865, + "loss": 2.7891, + "theoretical_loss": 3.6295474917431965, + "tokens_seen": 1058668544 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430591775325978, + "loss": 2.7441, + "theoretical_loss": 3.6295263348132822, + "tokens_seen": 1058734080 + }, + { + "epoch": 0.32, + "learning_rate": 0.000343049147442327, + "loss": 2.3854, + "theoretical_loss": 3.629505179559617, + "tokens_seen": 1058799616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430391173520562, + "loss": 2.6415, + "theoretical_loss": 3.6294840259819643, + "tokens_seen": 1058865152 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003430290872617854, + "loss": 2.8416, + "theoretical_loss": 3.6294628740800867, + "tokens_seen": 1058930688 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034301905717151456, + "loss": 2.6718, + "theoretical_loss": 3.6294417238537484, + "tokens_seen": 1058996224 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034300902708124374, + "loss": 2.5243, + "theoretical_loss": 3.6294205753027127, + "tokens_seen": 1059061760 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429989969909729, + "loss": 2.6848, + "theoretical_loss": 3.6293994284267437, + "tokens_seen": 1059127296 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034298896690070215, + "loss": 2.7442, + "theoretical_loss": 3.6293782832256047, + "tokens_seen": 1059192832 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429789368104313, + "loss": 2.6438, + "theoretical_loss": 3.629357139699059, + "tokens_seen": 1059258368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429689067201605, + "loss": 2.7197, + "theoretical_loss": 3.629335997846872, + "tokens_seen": 1059323904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429588766298897, + "loss": 2.4123, + "theoretical_loss": 3.6293148576688057, + "tokens_seen": 1059389440 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429488465396189, + "loss": 2.662, + "theoretical_loss": 3.6292937191646253, + "tokens_seen": 1059454976 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034293881644934806, + "loss": 2.433, + "theoretical_loss": 3.6292725823340932, + "tokens_seen": 1059520512 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034292878635907724, + "loss": 2.5597, + "theoretical_loss": 3.6292514471769755, + "tokens_seen": 1059586048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003429187562688064, + "loss": 2.9441, + "theoretical_loss": 3.629230313693035, + "tokens_seen": 1059651584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034290872617853566, + "loss": 2.3666, + "theoretical_loss": 3.629209181882035, + "tokens_seen": 1059717120 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428986960882648, + "loss": 2.9156, + "theoretical_loss": 3.629188051743741, + "tokens_seen": 1059782656 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342888665997994, + "loss": 2.6805, + "theoretical_loss": 3.6291669232779165, + "tokens_seen": 1059848192 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034287863590772315, + "loss": 2.5692, + "theoretical_loss": 3.629145796484326, + "tokens_seen": 1059913728 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428686058174524, + "loss": 2.7361, + "theoretical_loss": 3.6291246713627334, + "tokens_seen": 1059979264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 592062, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5544235706329346, + "objective/train/theoretical_loss": 3.6291035479129032, + "objective/train/tokens_used": 1080504800, + "theoretical_loss": 3.6291035479129032, + "tokens_seen": 1060044800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034285857572718156, + "loss": 2.4871, + "theoretical_loss": 3.6291035479129032, + "tokens_seen": 1060044800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034284854563691074, + "loss": 2.3404, + "theoretical_loss": 3.6290824261346, + "tokens_seen": 1060110336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428385155466399, + "loss": 2.8714, + "theoretical_loss": 3.6290613060275883, + "tokens_seen": 1060175872 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428284854563691, + "loss": 2.7667, + "theoretical_loss": 3.629040187591632, + "tokens_seen": 1060241408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428184553660983, + "loss": 2.7177, + "theoretical_loss": 3.6290190708264958, + "tokens_seen": 1060306944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003428084252758275, + "loss": 2.7029, + "theoretical_loss": 3.6289979557319443, + "tokens_seen": 1060372480 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034279839518555665, + "loss": 2.634, + "theoretical_loss": 3.628976842307742, + "tokens_seen": 1060438016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427883650952859, + "loss": 2.5498, + "theoretical_loss": 3.6289557305536544, + "tokens_seen": 1060503552 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034277833500501507, + "loss": 2.6537, + "theoretical_loss": 3.628934620469445, + "tokens_seen": 1060569088 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034276830491474425, + "loss": 2.7612, + "theoretical_loss": 3.6289135120548783, + "tokens_seen": 1060634624 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034275827482447343, + "loss": 2.845, + "theoretical_loss": 3.6288924053097205, + "tokens_seen": 1060700160 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427482447342026, + "loss": 2.8157, + "theoretical_loss": 3.628871300233736, + "tokens_seen": 1060765696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427382146439318, + "loss": 2.9871, + "theoretical_loss": 3.628850196826689, + "tokens_seen": 1060831232 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342728184553661, + "loss": 3.0334, + "theoretical_loss": 3.628829095088345, + "tokens_seen": 1060896768 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034271815446339015, + "loss": 2.5297, + "theoretical_loss": 3.6288079950184686, + "tokens_seen": 1060962304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003427081243731194, + "loss": 2.5689, + "theoretical_loss": 3.628786896616825, + "tokens_seen": 1061027840 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426980942828485, + "loss": 2.7144, + "theoretical_loss": 3.6287657998831797, + "tokens_seen": 1061093376 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034268806419257775, + "loss": 2.6815, + "theoretical_loss": 3.628744704817297, + "tokens_seen": 1061158912 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034267803410230693, + "loss": 2.7748, + "theoretical_loss": 3.628723611418943, + "tokens_seen": 1061224448 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426680040120361, + "loss": 2.6297, + "theoretical_loss": 3.6287025196878826, + "tokens_seen": 1061289984 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426579739217653, + "loss": 2.5459, + "theoretical_loss": 3.6286814296238803, + "tokens_seen": 1061355520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034264794383149453, + "loss": 2.872, + "theoretical_loss": 3.6286603412267024, + "tokens_seen": 1061421056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034263791374122366, + "loss": 2.6676, + "theoretical_loss": 3.6286392544961137, + "tokens_seen": 1061486592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003426278836509529, + "loss": 2.6477, + "theoretical_loss": 3.62861816943188, + "tokens_seen": 1061552128 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342617853560682, + "loss": 2.8712, + "theoretical_loss": 3.6285970860337664, + "tokens_seen": 1061617664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 592448, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1389501094818115, + "objective/train/theoretical_loss": 3.6285760043015385, + "objective/train/tokens_used": 1082143200, + "theoretical_loss": 3.6285760043015385, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034260782347041125, + "loss": 2.7504, + "theoretical_loss": 3.6285760043015385, + "tokens_seen": 1061683200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034259779338014043, + "loss": 2.6796, + "theoretical_loss": 3.6285549242349626, + "tokens_seen": 1061748736 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425877632898696, + "loss": 2.6368, + "theoretical_loss": 3.628533845833803, + "tokens_seen": 1061814272 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425777331995988, + "loss": 2.8308, + "theoretical_loss": 3.628512769097826, + "tokens_seen": 1061879808 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342567703109328, + "loss": 2.613, + "theoretical_loss": 3.6284916940267973, + "tokens_seen": 1061945344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034255767301905716, + "loss": 2.9996, + "theoretical_loss": 3.628470620620482, + "tokens_seen": 1062010880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425476429287864, + "loss": 2.6594, + "theoretical_loss": 3.6284495488786472, + "tokens_seen": 1062076416 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425376128385155, + "loss": 2.523, + "theoretical_loss": 3.628428478801058, + "tokens_seen": 1062141952 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034252758274824476, + "loss": 2.87, + "theoretical_loss": 3.62840741038748, + "tokens_seen": 1062207488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425175526579739, + "loss": 2.608, + "theoretical_loss": 3.62838634363768, + "tokens_seen": 1062273024 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003425075225677031, + "loss": 2.6211, + "theoretical_loss": 3.628365278551423, + "tokens_seen": 1062338560 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424974924774323, + "loss": 2.8357, + "theoretical_loss": 3.6283442151284753, + "tokens_seen": 1062404096 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424874623871615, + "loss": 2.7023, + "theoretical_loss": 3.628323153368603, + "tokens_seen": 1062469632 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034247743229689066, + "loss": 2.2979, + "theoretical_loss": 3.628302093271573, + "tokens_seen": 1062535168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424674022066199, + "loss": 2.5555, + "theoretical_loss": 3.6282810348371504, + "tokens_seen": 1062600704 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342457372116349, + "loss": 2.4632, + "theoretical_loss": 3.6282599780651017, + "tokens_seen": 1062666240 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034244734202607826, + "loss": 2.9378, + "theoretical_loss": 3.6282389229551932, + "tokens_seen": 1062731776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424373119358074, + "loss": 2.6977, + "theoretical_loss": 3.6282178695071914, + "tokens_seen": 1062797312 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424272818455366, + "loss": 2.4779, + "theoretical_loss": 3.6281968177208626, + "tokens_seen": 1062862848 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003424172517552658, + "loss": 2.7706, + "theoretical_loss": 3.628175767595973, + "tokens_seen": 1062928384 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342407221664995, + "loss": 2.7399, + "theoretical_loss": 3.628154719132289, + "tokens_seen": 1062993920 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034239719157472416, + "loss": 2.4667, + "theoretical_loss": 3.628133672329578, + "tokens_seen": 1063059456 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034238716148445335, + "loss": 2.7334, + "theoretical_loss": 3.628112627187605, + "tokens_seen": 1063124992 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423771313941825, + "loss": 2.8203, + "theoretical_loss": 3.628091583706138, + "tokens_seen": 1063190528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034236710130391176, + "loss": 2.614, + "theoretical_loss": 3.6280705418849433, + "tokens_seen": 1063256064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 593762, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.574810743331909, + "objective/train/theoretical_loss": 3.628049501723787, + "objective/train/tokens_used": 1083781600, + "theoretical_loss": 3.628049501723787, + "tokens_seen": 1063321600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423570712136409, + "loss": 2.7115, + "theoretical_loss": 3.628049501723787, + "tokens_seen": 1063321600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423470411233701, + "loss": 2.7239, + "theoretical_loss": 3.628028463222436, + "tokens_seen": 1063387136 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423370110330993, + "loss": 2.5938, + "theoretical_loss": 3.628007426380657, + "tokens_seen": 1063452672 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423269809428285, + "loss": 2.7494, + "theoretical_loss": 3.6279863911982178, + "tokens_seen": 1063518208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003423169508525577, + "loss": 2.5914, + "theoretical_loss": 3.627965357674884, + "tokens_seen": 1063583744 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034230692076228685, + "loss": 2.6498, + "theoretical_loss": 3.6279443258104234, + "tokens_seen": 1063649280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422968906720161, + "loss": 2.9348, + "theoretical_loss": 3.627923295604603, + "tokens_seen": 1063714816 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034228686058174527, + "loss": 2.6254, + "theoretical_loss": 3.6279022670571885, + "tokens_seen": 1063780352 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034227683049147445, + "loss": 2.7505, + "theoretical_loss": 3.627881240167949, + "tokens_seen": 1063845888 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034226680040120363, + "loss": 2.6287, + "theoretical_loss": 3.62786021493665, + "tokens_seen": 1063911424 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422567703109328, + "loss": 2.6029, + "theoretical_loss": 3.6278391913630594, + "tokens_seen": 1063976960 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342246740220662, + "loss": 2.8587, + "theoretical_loss": 3.627818169446944, + "tokens_seen": 1064042496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422367101303912, + "loss": 2.7094, + "theoretical_loss": 3.6277971491880714, + "tokens_seen": 1064108032 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034222668004012035, + "loss": 2.5434, + "theoretical_loss": 3.627776130586209, + "tokens_seen": 1064173568 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422166499498496, + "loss": 2.9045, + "theoretical_loss": 3.6277551136411237, + "tokens_seen": 1064239104 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003422066198595787, + "loss": 2.8314, + "theoretical_loss": 3.6277340983525828, + "tokens_seen": 1064304640 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034219658976930795, + "loss": 2.473, + "theoretical_loss": 3.6277130847203543, + "tokens_seen": 1064370176 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034218655967903713, + "loss": 2.6314, + "theoretical_loss": 3.627692072744206, + "tokens_seen": 1064435712 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421765295887663, + "loss": 2.4801, + "theoretical_loss": 3.6276710624239037, + "tokens_seen": 1064501248 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421664994984955, + "loss": 2.6896, + "theoretical_loss": 3.627650053759217, + "tokens_seen": 1064566784 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034215646940822473, + "loss": 2.8167, + "theoretical_loss": 3.6276290467499126, + "tokens_seen": 1064632320 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034214643931795386, + "loss": 2.6443, + "theoretical_loss": 3.6276080413957583, + "tokens_seen": 1064697856 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421364092276831, + "loss": 2.8321, + "theoretical_loss": 3.6275870376965216, + "tokens_seen": 1064763392 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003421263791374122, + "loss": 2.511, + "theoretical_loss": 3.62756603565197, + "tokens_seen": 1064828928 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034211634904714145, + "loss": 2.6694, + "theoretical_loss": 3.6275450352618726, + "tokens_seen": 1064894464 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 594388, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.325084686279297, + "objective/train/theoretical_loss": 3.6275240365259958, + "objective/train/tokens_used": 1085420000, + "theoretical_loss": 3.6275240365259958, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034210631895687063, + "loss": 2.5833, + "theoretical_loss": 3.6275240365259958, + "tokens_seen": 1064960000 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420962888665998, + "loss": 2.7116, + "theoretical_loss": 3.6275030394441083, + "tokens_seen": 1065025536 + }, + { + "epoch": 0.32, + "learning_rate": 0.000342086258776329, + "loss": 2.5262, + "theoretical_loss": 3.627482044015978, + "tokens_seen": 1065091072 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420762286860582, + "loss": 2.6584, + "theoretical_loss": 3.627461050241373, + "tokens_seen": 1065156608 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034206619859578736, + "loss": 2.6601, + "theoretical_loss": 3.6274400581200608, + "tokens_seen": 1065222144 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420561685055166, + "loss": 2.6521, + "theoretical_loss": 3.62741906765181, + "tokens_seen": 1065287680 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420461384152457, + "loss": 2.5302, + "theoretical_loss": 3.6273980788363884, + "tokens_seen": 1065353216 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034203610832497496, + "loss": 2.4866, + "theoretical_loss": 3.6273770916735644, + "tokens_seen": 1065418752 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420260782347041, + "loss": 2.614, + "theoretical_loss": 3.6273561061631066, + "tokens_seen": 1065484288 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420160481444333, + "loss": 2.713, + "theoretical_loss": 3.6273351223047823, + "tokens_seen": 1065549824 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003420060180541625, + "loss": 2.6868, + "theoretical_loss": 3.6273141400983606, + "tokens_seen": 1065615360 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419959879638917, + "loss": 2.4458, + "theoretical_loss": 3.6272931595436098, + "tokens_seen": 1065680896 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034198595787362086, + "loss": 2.6685, + "theoretical_loss": 3.6272721806402983, + "tokens_seen": 1065746432 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419759277833501, + "loss": 2.6327, + "theoretical_loss": 3.6272512033881945, + "tokens_seen": 1065811968 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419658976930792, + "loss": 2.6649, + "theoretical_loss": 3.6272302277870665, + "tokens_seen": 1065877504 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034195586760280846, + "loss": 2.6261, + "theoretical_loss": 3.627209253836684, + "tokens_seen": 1065943040 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419458375125376, + "loss": 2.7778, + "theoretical_loss": 3.627188281536814, + "tokens_seen": 1066008576 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419358074222668, + "loss": 2.6386, + "theoretical_loss": 3.627167310887227, + "tokens_seen": 1066074112 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341925777331996, + "loss": 2.6872, + "theoretical_loss": 3.6271463418876904, + "tokens_seen": 1066139648 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003419157472417252, + "loss": 2.7311, + "theoretical_loss": 3.6271253745379726, + "tokens_seen": 1066205184 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034190571715145436, + "loss": 2.6386, + "theoretical_loss": 3.6271044088378437, + "tokens_seen": 1066270720 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034189568706118355, + "loss": 2.5569, + "theoretical_loss": 3.627083444787072, + "tokens_seen": 1066336256 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418856569709127, + "loss": 2.5788, + "theoretical_loss": 3.6270624823854263, + "tokens_seen": 1066401792 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034187562688064196, + "loss": 2.4098, + "theoretical_loss": 3.6270415216326755, + "tokens_seen": 1066467328 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418655967903711, + "loss": 2.4781, + "theoretical_loss": 3.6270205625285885, + "tokens_seen": 1066532864 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 595571, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6795873641967773, + "objective/train/theoretical_loss": 3.6269996050729345, + "objective/train/tokens_used": 1087058400, + "theoretical_loss": 3.6269996050729345, + "tokens_seen": 1066598400 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418555667001003, + "loss": 2.613, + "theoretical_loss": 3.6269996050729345, + "tokens_seen": 1066598400 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034184553660982945, + "loss": 2.5394, + "theoretical_loss": 3.6269786492654825, + "tokens_seen": 1066663936 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003418355065195587, + "loss": 2.3815, + "theoretical_loss": 3.626957695106002, + "tokens_seen": 1066729472 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034182547642928787, + "loss": 2.4471, + "theoretical_loss": 3.6269367425942614, + "tokens_seen": 1066795008 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034181544633901705, + "loss": 2.6606, + "theoretical_loss": 3.626915791730031, + "tokens_seen": 1066860544 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034180541624874623, + "loss": 2.637, + "theoretical_loss": 3.626894842513079, + "tokens_seen": 1066926080 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034179538615847547, + "loss": 2.8521, + "theoretical_loss": 3.6268738949431754, + "tokens_seen": 1066991616 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417853560682046, + "loss": 2.5181, + "theoretical_loss": 3.6268529490200887, + "tokens_seen": 1067057152 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034177532597793383, + "loss": 2.6079, + "theoretical_loss": 3.626832004743589, + "tokens_seen": 1067122688 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034176529588766295, + "loss": 2.6194, + "theoretical_loss": 3.626811062113446, + "tokens_seen": 1067188224 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417552657973922, + "loss": 2.6109, + "theoretical_loss": 3.626790121129429, + "tokens_seen": 1067253760 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034174523570712137, + "loss": 2.6758, + "theoretical_loss": 3.626769181791307, + "tokens_seen": 1067319296 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034173520561685055, + "loss": 2.6405, + "theoretical_loss": 3.62674824409885, + "tokens_seen": 1067384832 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034172517552657973, + "loss": 2.7384, + "theoretical_loss": 3.626727308051828, + "tokens_seen": 1067450368 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417151454363089, + "loss": 2.7122, + "theoretical_loss": 3.6267063736500096, + "tokens_seen": 1067515904 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003417051153460381, + "loss": 2.741, + "theoretical_loss": 3.6266854408931657, + "tokens_seen": 1067581440 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034169508525576733, + "loss": 2.5662, + "theoretical_loss": 3.6266645097810652, + "tokens_seen": 1067646976 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034168505516549646, + "loss": 2.4996, + "theoretical_loss": 3.6266435803134787, + "tokens_seen": 1067712512 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416750250752257, + "loss": 2.5729, + "theoretical_loss": 3.6266226524901755, + "tokens_seen": 1067778048 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416649949849548, + "loss": 2.738, + "theoretical_loss": 3.6266017263109256, + "tokens_seen": 1067843584 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034165496489468406, + "loss": 2.7026, + "theoretical_loss": 3.626580801775499, + "tokens_seen": 1067909120 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034164493480441324, + "loss": 2.3374, + "theoretical_loss": 3.6265598788836657, + "tokens_seen": 1067974656 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416349047141424, + "loss": 2.6919, + "theoretical_loss": 3.626538957635196, + "tokens_seen": 1068040192 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003416248746238716, + "loss": 2.8102, + "theoretical_loss": 3.62651803802986, + "tokens_seen": 1068105728 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034161484453360083, + "loss": 2.5649, + "theoretical_loss": 3.626497120067427, + "tokens_seen": 1068171264 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 595858, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9211819171905518, + "objective/train/theoretical_loss": 3.6264762037476683, + "objective/train/tokens_used": 1088696800, + "theoretical_loss": 3.6264762037476683, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034160481444332996, + "loss": 2.5103, + "theoretical_loss": 3.6264762037476683, + "tokens_seen": 1068236800 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415947843530592, + "loss": 2.6812, + "theoretical_loss": 3.6264552890703534, + "tokens_seen": 1068302336 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415847542627884, + "loss": 2.6637, + "theoretical_loss": 3.6264343760352533, + "tokens_seen": 1068367872 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034157472417251756, + "loss": 2.7339, + "theoretical_loss": 3.6264134646421375, + "tokens_seen": 1068433408 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415646940822468, + "loss": 2.6223, + "theoretical_loss": 3.6263925548907765, + "tokens_seen": 1068498944 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415546639919759, + "loss": 2.6919, + "theoretical_loss": 3.6263716467809415, + "tokens_seen": 1068564480 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034154463390170516, + "loss": 2.7286, + "theoretical_loss": 3.626350740312402, + "tokens_seen": 1068630016 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415346038114343, + "loss": 2.5326, + "theoretical_loss": 3.6263298354849294, + "tokens_seen": 1068695552 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415245737211635, + "loss": 2.5669, + "theoretical_loss": 3.6263089322982935, + "tokens_seen": 1068761088 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415145436308927, + "loss": 2.656, + "theoretical_loss": 3.6262880307522654, + "tokens_seen": 1068826624 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003415045135406219, + "loss": 2.5146, + "theoretical_loss": 3.626267130846615, + "tokens_seen": 1068892160 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034149448345035106, + "loss": 2.5452, + "theoretical_loss": 3.626246232581114, + "tokens_seen": 1068957696 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414844533600803, + "loss": 2.5444, + "theoretical_loss": 3.6262253359555325, + "tokens_seen": 1069023232 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414744232698094, + "loss": 2.507, + "theoretical_loss": 3.6262044409696417, + "tokens_seen": 1069088768 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034146439317953866, + "loss": 2.5204, + "theoretical_loss": 3.6261835476232123, + "tokens_seen": 1069154304 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414543630892678, + "loss": 2.6365, + "theoretical_loss": 3.6261626559160147, + "tokens_seen": 1069219840 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341444332998997, + "loss": 2.5105, + "theoretical_loss": 3.6261417658478208, + "tokens_seen": 1069285376 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414343029087262, + "loss": 2.7935, + "theoretical_loss": 3.6261208774184004, + "tokens_seen": 1069350912 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003414242728184554, + "loss": 2.575, + "theoretical_loss": 3.6260999906275257, + "tokens_seen": 1069416448 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034141424272818456, + "loss": 2.5472, + "theoretical_loss": 3.6260791054749673, + "tokens_seen": 1069481984 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034140421263791375, + "loss": 2.5069, + "theoretical_loss": 3.6260582219604958, + "tokens_seen": 1069547520 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034139418254764293, + "loss": 2.7579, + "theoretical_loss": 3.626037340083883, + "tokens_seen": 1069613056 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034138415245737216, + "loss": 2.6508, + "theoretical_loss": 3.6260164598448994, + "tokens_seen": 1069678592 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413741223671013, + "loss": 2.7109, + "theoretical_loss": 3.625995581243317, + "tokens_seen": 1069744128 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413640922768305, + "loss": 2.6212, + "theoretical_loss": 3.6259747042789074, + "tokens_seen": 1069809664 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 597140, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7327358722686768, + "objective/train/theoretical_loss": 3.625953828951441, + "objective/train/tokens_used": 1090335200, + "theoretical_loss": 3.625953828951441, + "tokens_seen": 1069875200 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034135406218655965, + "loss": 2.6413, + "theoretical_loss": 3.625953828951441, + "tokens_seen": 1069875200 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003413440320962889, + "loss": 2.6578, + "theoretical_loss": 3.625932955260689, + "tokens_seen": 1069940736 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034133400200601807, + "loss": 2.6887, + "theoretical_loss": 3.625912083206424, + "tokens_seen": 1070006272 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034132397191574725, + "loss": 2.3918, + "theoretical_loss": 3.625891212788417, + "tokens_seen": 1070071808 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034131394182547643, + "loss": 2.8379, + "theoretical_loss": 3.625870344006439, + "tokens_seen": 1070137344 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034130391173520567, + "loss": 2.8266, + "theoretical_loss": 3.625849476860262, + "tokens_seen": 1070202880 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412938816449348, + "loss": 2.6852, + "theoretical_loss": 3.6258286113496574, + "tokens_seen": 1070268416 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034128385155466403, + "loss": 2.5112, + "theoretical_loss": 3.6258077474743975, + "tokens_seen": 1070333952 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034127382146439315, + "loss": 2.6649, + "theoretical_loss": 3.625786885234253, + "tokens_seen": 1070399488 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412637913741224, + "loss": 2.5514, + "theoretical_loss": 3.625766024628997, + "tokens_seen": 1070465024 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034125376128385157, + "loss": 2.7262, + "theoretical_loss": 3.6257451656584, + "tokens_seen": 1070530560 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034124373119358075, + "loss": 2.6269, + "theoretical_loss": 3.6257243083222344, + "tokens_seen": 1070596096 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034123370110330993, + "loss": 2.5139, + "theoretical_loss": 3.6257034526202725, + "tokens_seen": 1070661632 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412236710130391, + "loss": 2.7592, + "theoretical_loss": 3.6256825985522854, + "tokens_seen": 1070727168 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003412136409227683, + "loss": 2.6409, + "theoretical_loss": 3.6256617461180456, + "tokens_seen": 1070792704 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034120361083249753, + "loss": 2.6633, + "theoretical_loss": 3.625640895317325, + "tokens_seen": 1070858240 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034119358074222666, + "loss": 2.5714, + "theoretical_loss": 3.625620046149895, + "tokens_seen": 1070923776 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411835506519559, + "loss": 2.6975, + "theoretical_loss": 3.6255991986155296, + "tokens_seen": 1070989312 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341173520561685, + "loss": 2.8864, + "theoretical_loss": 3.625578352713999, + "tokens_seen": 1071054848 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034116349047141426, + "loss": 2.8835, + "theoretical_loss": 3.625557508445076, + "tokens_seen": 1071120384 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034115346038114344, + "loss": 2.434, + "theoretical_loss": 3.625536665808534, + "tokens_seen": 1071185920 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411434302908726, + "loss": 2.7717, + "theoretical_loss": 3.6255158248041433, + "tokens_seen": 1071251456 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411334002006018, + "loss": 2.6301, + "theoretical_loss": 3.6254949854316783, + "tokens_seen": 1071316992 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034112337011033103, + "loss": 2.6367, + "theoretical_loss": 3.62547414769091, + "tokens_seen": 1071382528 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034111334002006016, + "loss": 2.5804, + "theoretical_loss": 3.6254533115816105, + "tokens_seen": 1071448064 + }, + { + "epoch": 0.32, + "objective/train/docs_used": 598183, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3538804054260254, + "objective/train/theoretical_loss": 3.625432477103554, + "objective/train/tokens_used": 1091973600, + "theoretical_loss": 3.625432477103554, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003411033099297894, + "loss": 2.3987, + "theoretical_loss": 3.625432477103554, + "tokens_seen": 1071513600 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410932798395185, + "loss": 2.6763, + "theoretical_loss": 3.6254116442565114, + "tokens_seen": 1071579136 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034108324974924776, + "loss": 2.8075, + "theoretical_loss": 3.625390813040257, + "tokens_seen": 1071644672 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034107321965897694, + "loss": 2.6814, + "theoretical_loss": 3.625369983454561, + "tokens_seen": 1071710208 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410631895687061, + "loss": 2.6329, + "theoretical_loss": 3.6253491554991983, + "tokens_seen": 1071775744 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410531594784353, + "loss": 2.3829, + "theoretical_loss": 3.62532832917394, + "tokens_seen": 1071841280 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410431293881645, + "loss": 2.6071, + "theoretical_loss": 3.6253075044785605, + "tokens_seen": 1071906816 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034103309929789366, + "loss": 2.7892, + "theoretical_loss": 3.625286681412831, + "tokens_seen": 1071972352 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003410230692076229, + "loss": 2.7847, + "theoretical_loss": 3.6252658599765257, + "tokens_seen": 1072037888 + }, + { + "epoch": 0.32, + "learning_rate": 0.000341013039117352, + "loss": 2.6912, + "theoretical_loss": 3.6252450401694167, + "tokens_seen": 1072103424 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034100300902708126, + "loss": 2.5225, + "theoretical_loss": 3.6252242219912767, + "tokens_seen": 1072168960 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034099297893681044, + "loss": 2.6905, + "theoretical_loss": 3.6252034054418796, + "tokens_seen": 1072234496 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409829488465396, + "loss": 2.7699, + "theoretical_loss": 3.6251825905209976, + "tokens_seen": 1072300032 + }, + { + "epoch": 0.32, + "learning_rate": 0.0003409729187562688, + "loss": 2.4818, + "theoretical_loss": 3.625161777228404, + "tokens_seen": 1072365568 + }, + { + "epoch": 0.32, + "learning_rate": 0.000340962888665998, + "loss": 2.8261, + "theoretical_loss": 3.6251409655638724, + "tokens_seen": 1072431104 + }, + { + "epoch": 0.32, + "learning_rate": 0.00034095285857572717, + "loss": 2.7078, + "theoretical_loss": 3.6251201555271755, + "tokens_seen": 1072496640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003409428284854564, + "loss": 2.5445, + "theoretical_loss": 3.625099347118087, + "tokens_seen": 1072562176 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034093279839518553, + "loss": 2.6212, + "theoretical_loss": 3.625078540336379, + "tokens_seen": 1072627712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034092276830491477, + "loss": 2.5129, + "theoretical_loss": 3.625057735181826, + "tokens_seen": 1072693248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003409127382146439, + "loss": 2.6918, + "theoretical_loss": 3.6250369316542006, + "tokens_seen": 1072758784 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034090270812437313, + "loss": 2.4667, + "theoretical_loss": 3.625016129753277, + "tokens_seen": 1072824320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408926780341023, + "loss": 2.4542, + "theoretical_loss": 3.6249953294788284, + "tokens_seen": 1072889856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003408826479438315, + "loss": 2.6622, + "theoretical_loss": 3.6249745308306274, + "tokens_seen": 1072955392 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034087261785356067, + "loss": 2.7896, + "theoretical_loss": 3.6249537338084488, + "tokens_seen": 1073020928 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034086258776328985, + "loss": 2.7108, + "theoretical_loss": 3.624932938412065, + "tokens_seen": 1073086464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 598781, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9608399868011475, + "objective/train/theoretical_loss": 3.6249121446412507, + "objective/train/tokens_used": 1093612000, + "theoretical_loss": 3.6249121446412507, + "tokens_seen": 1073152000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034085255767301903, + "loss": 2.7379, + "theoretical_loss": 3.6249121446412507, + "tokens_seen": 1073152000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034084252758274827, + "loss": 2.5467, + "theoretical_loss": 3.624891352495779, + "tokens_seen": 1073217536 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034083249749247745, + "loss": 2.3039, + "theoretical_loss": 3.624870561975423, + "tokens_seen": 1073283072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034082246740220663, + "loss": 2.6401, + "theoretical_loss": 3.6248497730799585, + "tokens_seen": 1073348608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034081243731193587, + "loss": 2.4763, + "theoretical_loss": 3.624828985809157, + "tokens_seen": 1073414144 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340802407221665, + "loss": 2.5679, + "theoretical_loss": 3.6248082001627937, + "tokens_seen": 1073479680 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034079237713139423, + "loss": 2.7953, + "theoretical_loss": 3.6247874161406415, + "tokens_seen": 1073545216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034078234704112335, + "loss": 2.6214, + "theoretical_loss": 3.6247666337424755, + "tokens_seen": 1073610752 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407723169508526, + "loss": 2.6529, + "theoretical_loss": 3.6247458529680694, + "tokens_seen": 1073676288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034076228686058177, + "loss": 2.7148, + "theoretical_loss": 3.6247250738171966, + "tokens_seen": 1073741824 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034075225677031095, + "loss": 2.9034, + "theoretical_loss": 3.6247042962896314, + "tokens_seen": 1073807360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034074222668004013, + "loss": 2.6231, + "theoretical_loss": 3.6246835203851484, + "tokens_seen": 1073872896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407321965897693, + "loss": 2.6436, + "theoretical_loss": 3.624662746103521, + "tokens_seen": 1073938432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003407221664994985, + "loss": 2.5244, + "theoretical_loss": 3.6246419734445245, + "tokens_seen": 1074003968 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034071213640922773, + "loss": 2.6201, + "theoretical_loss": 3.624621202407932, + "tokens_seen": 1074069504 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034070210631895686, + "loss": 2.5998, + "theoretical_loss": 3.6246004329935184, + "tokens_seen": 1074135040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406920762286861, + "loss": 2.4807, + "theoretical_loss": 3.6245796652010585, + "tokens_seen": 1074200576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406820461384152, + "loss": 2.7288, + "theoretical_loss": 3.6245588990303252, + "tokens_seen": 1074266112 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034067201604814446, + "loss": 2.4171, + "theoretical_loss": 3.624538134481095, + "tokens_seen": 1074331648 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034066198595787364, + "loss": 2.6719, + "theoretical_loss": 3.6245173715531402, + "tokens_seen": 1074397184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406519558676028, + "loss": 2.6694, + "theoretical_loss": 3.6244966102462373, + "tokens_seen": 1074462720 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340641925777332, + "loss": 2.8719, + "theoretical_loss": 3.6244758505601595, + "tokens_seen": 1074528256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034063189568706123, + "loss": 2.687, + "theoretical_loss": 3.624455092494682, + "tokens_seen": 1074593792 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034062186559679036, + "loss": 2.7869, + "theoretical_loss": 3.624434336049579, + "tokens_seen": 1074659328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406118355065196, + "loss": 2.5128, + "theoretical_loss": 3.6244135812246263, + "tokens_seen": 1074724864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 599993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5279252529144287, + "objective/train/theoretical_loss": 3.6243928280195976, + "objective/train/tokens_used": 1095250400, + "theoretical_loss": 3.6243928280195976, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003406018054162487, + "loss": 2.6169, + "theoretical_loss": 3.6243928280195976, + "tokens_seen": 1074790400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034059177532597796, + "loss": 2.6747, + "theoretical_loss": 3.624372076434268, + "tokens_seen": 1074855936 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034058174523570714, + "loss": 2.7016, + "theoretical_loss": 3.6243513264684126, + "tokens_seen": 1074921472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405717151454363, + "loss": 2.7665, + "theoretical_loss": 3.624330578121806, + "tokens_seen": 1074987008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405616850551655, + "loss": 2.859, + "theoretical_loss": 3.624309831394223, + "tokens_seen": 1075052544 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405516549648947, + "loss": 2.8091, + "theoretical_loss": 3.624289086285439, + "tokens_seen": 1075118080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034054162487462386, + "loss": 2.787, + "theoretical_loss": 3.6242683427952285, + "tokens_seen": 1075183616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405315947843531, + "loss": 2.6744, + "theoretical_loss": 3.6242476009233675, + "tokens_seen": 1075249152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003405215646940822, + "loss": 2.6618, + "theoretical_loss": 3.62422686066963, + "tokens_seen": 1075314688 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034051153460381146, + "loss": 2.4714, + "theoretical_loss": 3.6242061220337916, + "tokens_seen": 1075380224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034050150451354064, + "loss": 2.6045, + "theoretical_loss": 3.624185385015628, + "tokens_seen": 1075445760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404914744232698, + "loss": 2.5883, + "theoretical_loss": 3.6241646496149134, + "tokens_seen": 1075511296 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340481444332999, + "loss": 2.7271, + "theoretical_loss": 3.624143915831424, + "tokens_seen": 1075576832 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404714142427282, + "loss": 2.7838, + "theoretical_loss": 3.624123183664935, + "tokens_seen": 1075642368 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034046138415245737, + "loss": 2.7861, + "theoretical_loss": 3.6241024531152215, + "tokens_seen": 1075707904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404513540621866, + "loss": 2.7807, + "theoretical_loss": 3.6240817241820587, + "tokens_seen": 1075773440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034044132397191573, + "loss": 2.7066, + "theoretical_loss": 3.624060996865223, + "tokens_seen": 1075838976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034043129388164497, + "loss": 2.6692, + "theoretical_loss": 3.624040271164489, + "tokens_seen": 1075904512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404212637913741, + "loss": 2.6054, + "theoretical_loss": 3.624019547079633, + "tokens_seen": 1075970048 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034041123370110333, + "loss": 2.7437, + "theoretical_loss": 3.6239988246104295, + "tokens_seen": 1076035584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003404012036108325, + "loss": 2.651, + "theoretical_loss": 3.6239781037566554, + "tokens_seen": 1076101120 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403911735205617, + "loss": 2.6356, + "theoretical_loss": 3.6239573845180857, + "tokens_seen": 1076166656 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034038114343029087, + "loss": 2.4081, + "theoretical_loss": 3.6239366668944966, + "tokens_seen": 1076232192 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034037111334002005, + "loss": 2.6507, + "theoretical_loss": 3.6239159508856638, + "tokens_seen": 1076297728 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034036108324974923, + "loss": 2.6173, + "theoretical_loss": 3.6238952364913626, + "tokens_seen": 1076363264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 601033, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.560241460800171, + "objective/train/theoretical_loss": 3.623874523711369, + "objective/train/tokens_used": 1096888800, + "theoretical_loss": 3.623874523711369, + "tokens_seen": 1076428800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034035105315947847, + "loss": 2.4641, + "theoretical_loss": 3.623874523711369, + "tokens_seen": 1076428800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403410230692076, + "loss": 2.9075, + "theoretical_loss": 3.6238538125454594, + "tokens_seen": 1076494336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034033099297893683, + "loss": 2.7106, + "theoretical_loss": 3.623833102993409, + "tokens_seen": 1076559872 + }, + { + "epoch": 0.33, + "learning_rate": 0.000340320962888666, + "loss": 2.5626, + "theoretical_loss": 3.623812395054995, + "tokens_seen": 1076625408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403109327983952, + "loss": 2.667, + "theoretical_loss": 3.6237916887299924, + "tokens_seen": 1076690944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003403009027081244, + "loss": 2.4823, + "theoretical_loss": 3.623770984018178, + "tokens_seen": 1076756480 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034029087261785356, + "loss": 2.514, + "theoretical_loss": 3.623750280919327, + "tokens_seen": 1076822016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034028084252758274, + "loss": 2.8244, + "theoretical_loss": 3.623729579433217, + "tokens_seen": 1076887552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034027081243731197, + "loss": 2.6843, + "theoretical_loss": 3.623708879559623, + "tokens_seen": 1076953088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402607823470411, + "loss": 2.663, + "theoretical_loss": 3.6236881812983217, + "tokens_seen": 1077018624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034025075225677033, + "loss": 2.7929, + "theoretical_loss": 3.62366748464909, + "tokens_seen": 1077084160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034024072216649946, + "loss": 2.5873, + "theoretical_loss": 3.623646789611703, + "tokens_seen": 1077149696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402306920762287, + "loss": 2.5329, + "theoretical_loss": 3.623626096185938, + "tokens_seen": 1077215232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003402206619859579, + "loss": 2.8096, + "theoretical_loss": 3.6236054043715713, + "tokens_seen": 1077280768 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034021063189568706, + "loss": 2.4211, + "theoretical_loss": 3.6235847141683797, + "tokens_seen": 1077346304 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034020060180541624, + "loss": 2.6624, + "theoretical_loss": 3.623564025576139, + "tokens_seen": 1077411840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401905717151454, + "loss": 2.9951, + "theoretical_loss": 3.6235433385946267, + "tokens_seen": 1077477376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401805416248746, + "loss": 2.6661, + "theoretical_loss": 3.6235226532236187, + "tokens_seen": 1077542912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034017051153460384, + "loss": 2.5833, + "theoretical_loss": 3.623501969462892, + "tokens_seen": 1077608448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034016048144433296, + "loss": 2.7677, + "theoretical_loss": 3.623481287312223, + "tokens_seen": 1077673984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401504513540622, + "loss": 2.5585, + "theoretical_loss": 3.6234606067713893, + "tokens_seen": 1077739520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401404212637914, + "loss": 2.8325, + "theoretical_loss": 3.6234399278401668, + "tokens_seen": 1077805056 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034013039117352056, + "loss": 2.5578, + "theoretical_loss": 3.6234192505183325, + "tokens_seen": 1077870592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034012036108324974, + "loss": 2.8148, + "theoretical_loss": 3.623398574805664, + "tokens_seen": 1077936128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401103309929789, + "loss": 2.8187, + "theoretical_loss": 3.6233779007019375, + "tokens_seen": 1078001664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 601484, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.654538869857788, + "objective/train/theoretical_loss": 3.62335722820693, + "objective/train/tokens_used": 1098527200, + "theoretical_loss": 3.62335722820693, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003401003009027081, + "loss": 2.5519, + "theoretical_loss": 3.62335722820693, + "tokens_seen": 1078067200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034009027081243734, + "loss": 2.8683, + "theoretical_loss": 3.6233365573204193, + "tokens_seen": 1078132736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400802407221665, + "loss": 2.7965, + "theoretical_loss": 3.623315888042182, + "tokens_seen": 1078198272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400702106318957, + "loss": 2.8259, + "theoretical_loss": 3.623295220371995, + "tokens_seen": 1078263808 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400601805416249, + "loss": 2.5405, + "theoretical_loss": 3.623274554309636, + "tokens_seen": 1078329344 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034005015045135406, + "loss": 2.4842, + "theoretical_loss": 3.6232538898548814, + "tokens_seen": 1078394880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400401203610833, + "loss": 2.6957, + "theoretical_loss": 3.623233227007509, + "tokens_seen": 1078460416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003400300902708124, + "loss": 2.8692, + "theoretical_loss": 3.6232125657672962, + "tokens_seen": 1078525952 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034002006018054166, + "loss": 2.7149, + "theoretical_loss": 3.6231919061340205, + "tokens_seen": 1078591488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034001003009027084, + "loss": 2.6374, + "theoretical_loss": 3.6231712481074583, + "tokens_seen": 1078657024 + }, + { + "epoch": 0.33, + "learning_rate": 0.00034, + "loss": 2.5463, + "theoretical_loss": 3.6231505916873887, + "tokens_seen": 1078722560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399899699097292, + "loss": 2.6222, + "theoretical_loss": 3.623129936873587, + "tokens_seen": 1078788096 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399799398194584, + "loss": 2.9549, + "theoretical_loss": 3.623109283665833, + "tokens_seen": 1078853632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033996990972918757, + "loss": 2.5965, + "theoretical_loss": 3.6230886320639026, + "tokens_seen": 1078919168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399598796389168, + "loss": 2.7922, + "theoretical_loss": 3.6230679820675746, + "tokens_seen": 1078984704 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033994984954864593, + "loss": 2.7641, + "theoretical_loss": 3.6230473336766256, + "tokens_seen": 1079050240 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033993981945837517, + "loss": 2.5767, + "theoretical_loss": 3.6230266868908343, + "tokens_seen": 1079115776 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399297893681043, + "loss": 2.7457, + "theoretical_loss": 3.6230060417099774, + "tokens_seen": 1079181312 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033991975927783353, + "loss": 2.7767, + "theoretical_loss": 3.6229853981338334, + "tokens_seen": 1079246848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003399097291875627, + "loss": 2.5381, + "theoretical_loss": 3.62296475616218, + "tokens_seen": 1079312384 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398996990972919, + "loss": 2.7448, + "theoretical_loss": 3.622944115794795, + "tokens_seen": 1079377920 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033988966900702107, + "loss": 2.8126, + "theoretical_loss": 3.6229234770314562, + "tokens_seen": 1079443456 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033987963891675025, + "loss": 2.7071, + "theoretical_loss": 3.622902839871942, + "tokens_seen": 1079508992 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033986960882647943, + "loss": 2.8773, + "theoretical_loss": 3.6228822043160305, + "tokens_seen": 1079574528 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033985957873620867, + "loss": 2.6932, + "theoretical_loss": 3.622861570363499, + "tokens_seen": 1079640064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 602559, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.677448034286499, + "objective/train/theoretical_loss": 3.622840938014126, + "objective/train/tokens_used": 1100165600, + "theoretical_loss": 3.622840938014126, + "tokens_seen": 1079705600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398495486459378, + "loss": 2.611, + "theoretical_loss": 3.622840938014126, + "tokens_seen": 1079705600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033983951855566703, + "loss": 2.4328, + "theoretical_loss": 3.6228203072676894, + "tokens_seen": 1079771136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398294884653962, + "loss": 2.6645, + "theoretical_loss": 3.622799678123968, + "tokens_seen": 1079836672 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398194583751254, + "loss": 2.8604, + "theoretical_loss": 3.6227790505827393, + "tokens_seen": 1079902208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003398094282848546, + "loss": 2.6952, + "theoretical_loss": 3.6227584246437825, + "tokens_seen": 1079967744 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033979939819458376, + "loss": 2.6203, + "theoretical_loss": 3.622737800306875, + "tokens_seen": 1080033280 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033978936810431294, + "loss": 2.6824, + "theoretical_loss": 3.6227171775717952, + "tokens_seen": 1080098816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033977933801404217, + "loss": 2.4892, + "theoretical_loss": 3.6226965564383224, + "tokens_seen": 1080164352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397693079237713, + "loss": 2.8857, + "theoretical_loss": 3.6226759369062345, + "tokens_seen": 1080229888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033975927783350053, + "loss": 2.5533, + "theoretical_loss": 3.62265531897531, + "tokens_seen": 1080295424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033974924774322966, + "loss": 2.7272, + "theoretical_loss": 3.6226347026453274, + "tokens_seen": 1080360960 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397392176529589, + "loss": 2.8356, + "theoretical_loss": 3.622614087916065, + "tokens_seen": 1080426496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003397291875626881, + "loss": 2.6642, + "theoretical_loss": 3.622593474787302, + "tokens_seen": 1080492032 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033971915747241726, + "loss": 2.7828, + "theoretical_loss": 3.622572863258817, + "tokens_seen": 1080557568 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033970912738214644, + "loss": 2.7391, + "theoretical_loss": 3.6225522533303884, + "tokens_seen": 1080623104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396990972918756, + "loss": 2.7628, + "theoretical_loss": 3.622531645001795, + "tokens_seen": 1080688640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396890672016048, + "loss": 2.6882, + "theoretical_loss": 3.6225110382728163, + "tokens_seen": 1080754176 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033967903711133404, + "loss": 2.7996, + "theoretical_loss": 3.6224904331432297, + "tokens_seen": 1080819712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033966900702106316, + "loss": 2.74, + "theoretical_loss": 3.6224698296128155, + "tokens_seen": 1080885248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396589769307924, + "loss": 2.9426, + "theoretical_loss": 3.6224492276813516, + "tokens_seen": 1080950784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396489468405216, + "loss": 2.6184, + "theoretical_loss": 3.622428627348618, + "tokens_seen": 1081016320 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033963891675025076, + "loss": 2.6755, + "theoretical_loss": 3.6224080286143936, + "tokens_seen": 1081081856 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033962888665997994, + "loss": 2.8268, + "theoretical_loss": 3.6223874314784563, + "tokens_seen": 1081147392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396188565697091, + "loss": 2.6844, + "theoretical_loss": 3.622366835940586, + "tokens_seen": 1081212928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003396088264794383, + "loss": 2.6764, + "theoretical_loss": 3.622346242000562, + "tokens_seen": 1081278464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 603206, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.54484224319458, + "objective/train/theoretical_loss": 3.6223256496581637, + "objective/train/tokens_used": 1101804000, + "theoretical_loss": 3.6223256496581637, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033959879638916754, + "loss": 2.7759, + "theoretical_loss": 3.6223256496581637, + "tokens_seen": 1081344000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033958876629889667, + "loss": 2.7449, + "theoretical_loss": 3.62230505891317, + "tokens_seen": 1081409536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395787362086259, + "loss": 2.8069, + "theoretical_loss": 3.6222844697653596, + "tokens_seen": 1081475072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033956870611835503, + "loss": 2.567, + "theoretical_loss": 3.6222638822145132, + "tokens_seen": 1081540608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033955867602808426, + "loss": 2.7481, + "theoretical_loss": 3.6222432962604088, + "tokens_seen": 1081606144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033954864593781345, + "loss": 2.5326, + "theoretical_loss": 3.6222227119028263, + "tokens_seen": 1081671680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395386158475426, + "loss": 2.5933, + "theoretical_loss": 3.622202129141546, + "tokens_seen": 1081737216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003395285857572718, + "loss": 2.7438, + "theoretical_loss": 3.6221815479763464, + "tokens_seen": 1081802752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033951855566700104, + "loss": 2.4687, + "theoretical_loss": 3.622160968407007, + "tokens_seen": 1081868288 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033950852557673017, + "loss": 2.7814, + "theoretical_loss": 3.622140390433308, + "tokens_seen": 1081933824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394984954864594, + "loss": 2.5706, + "theoretical_loss": 3.622119814055029, + "tokens_seen": 1081999360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033948846539618853, + "loss": 2.7052, + "theoretical_loss": 3.6220992392719493, + "tokens_seen": 1082064896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033947843530591777, + "loss": 2.5586, + "theoretical_loss": 3.6220786660838487, + "tokens_seen": 1082130432 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033946840521564695, + "loss": 2.676, + "theoretical_loss": 3.622058094490507, + "tokens_seen": 1082195968 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033945837512537613, + "loss": 2.8831, + "theoretical_loss": 3.622037524491704, + "tokens_seen": 1082261504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394483450351053, + "loss": 2.6192, + "theoretical_loss": 3.6220169560872204, + "tokens_seen": 1082327040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394383149448345, + "loss": 2.6554, + "theoretical_loss": 3.6219963892768345, + "tokens_seen": 1082392576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394282848545637, + "loss": 2.5606, + "theoretical_loss": 3.6219758240603275, + "tokens_seen": 1082458112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003394182547642929, + "loss": 2.7442, + "theoretical_loss": 3.6219552604374785, + "tokens_seen": 1082523648 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033940822467402204, + "loss": 2.5597, + "theoretical_loss": 3.6219346984080687, + "tokens_seen": 1082589184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033939819458375127, + "loss": 2.7137, + "theoretical_loss": 3.621914137971877, + "tokens_seen": 1082654720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393881644934804, + "loss": 2.8618, + "theoretical_loss": 3.6218935791286846, + "tokens_seen": 1082720256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033937813440320963, + "loss": 2.5351, + "theoretical_loss": 3.6218730218782706, + "tokens_seen": 1082785792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393681043129388, + "loss": 2.7446, + "theoretical_loss": 3.6218524662204157, + "tokens_seen": 1082851328 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339358074222668, + "loss": 2.6969, + "theoretical_loss": 3.6218319121549003, + "tokens_seen": 1082916864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 603451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.421107530593872, + "objective/train/theoretical_loss": 3.6218113596815042, + "objective/train/tokens_used": 1103442400, + "theoretical_loss": 3.6218113596815042, + "tokens_seen": 1082982400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393480441323972, + "loss": 2.5057, + "theoretical_loss": 3.6218113596815042, + "tokens_seen": 1082982400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393380140421264, + "loss": 2.6102, + "theoretical_loss": 3.6217908088000086, + "tokens_seen": 1083047936 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393279839518556, + "loss": 2.8188, + "theoretical_loss": 3.6217702595101935, + "tokens_seen": 1083113472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003393179538615848, + "loss": 2.6628, + "theoretical_loss": 3.6217497118118382, + "tokens_seen": 1083179008 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033930792377131396, + "loss": 2.6231, + "theoretical_loss": 3.621729165704725, + "tokens_seen": 1083244544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033929789368104314, + "loss": 2.3983, + "theoretical_loss": 3.6217086211886333, + "tokens_seen": 1083310080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033928786359077237, + "loss": 2.6967, + "theoretical_loss": 3.621688078263344, + "tokens_seen": 1083375616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392778335005015, + "loss": 2.74, + "theoretical_loss": 3.6216675369286375, + "tokens_seen": 1083441152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033926780341023073, + "loss": 2.7629, + "theoretical_loss": 3.621646997184295, + "tokens_seen": 1083506688 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033925777331995986, + "loss": 2.6403, + "theoretical_loss": 3.6216264590300966, + "tokens_seen": 1083572224 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392477432296891, + "loss": 2.8355, + "theoretical_loss": 3.6216059224658226, + "tokens_seen": 1083637760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392377131394183, + "loss": 2.7227, + "theoretical_loss": 3.6215853874912547, + "tokens_seen": 1083703296 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033922768304914746, + "loss": 2.7424, + "theoretical_loss": 3.6215648541061736, + "tokens_seen": 1083768832 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033921765295887664, + "loss": 2.6606, + "theoretical_loss": 3.62154432231036, + "tokens_seen": 1083834368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003392076228686058, + "loss": 2.7373, + "theoretical_loss": 3.6215237921035945, + "tokens_seen": 1083899904 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339197592778335, + "loss": 2.5256, + "theoretical_loss": 3.6215032634856583, + "tokens_seen": 1083965440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033918756268806424, + "loss": 2.7521, + "theoretical_loss": 3.6214827364563327, + "tokens_seen": 1084030976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033917753259779336, + "loss": 2.7552, + "theoretical_loss": 3.6214622110153982, + "tokens_seen": 1084096512 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391675025075226, + "loss": 2.5513, + "theoretical_loss": 3.621441687162636, + "tokens_seen": 1084162048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391574724172518, + "loss": 2.7222, + "theoretical_loss": 3.6214211648978276, + "tokens_seen": 1084227584 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033914744232698096, + "loss": 2.6685, + "theoretical_loss": 3.6214006442207536, + "tokens_seen": 1084293120 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033913741223671014, + "loss": 2.5153, + "theoretical_loss": 3.6213801251311954, + "tokens_seen": 1084358656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391273821464393, + "loss": 2.5689, + "theoretical_loss": 3.6213596076289347, + "tokens_seen": 1084424192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003391173520561685, + "loss": 2.7692, + "theoretical_loss": 3.6213390917137525, + "tokens_seen": 1084489728 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033910732196589774, + "loss": 2.5669, + "theoretical_loss": 3.6213185773854297, + "tokens_seen": 1084555264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 603451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2022194862365723, + "objective/train/theoretical_loss": 3.6212980646437485, + "objective/train/tokens_used": 1105080800, + "theoretical_loss": 3.6212980646437485, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033909729187562687, + "loss": 2.8285, + "theoretical_loss": 3.6212980646437485, + "tokens_seen": 1084620800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003390872617853561, + "loss": 2.5917, + "theoretical_loss": 3.62127755348849, + "tokens_seen": 1084686336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033907723169508523, + "loss": 2.5659, + "theoretical_loss": 3.621257043919435, + "tokens_seen": 1084751872 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033906720160481446, + "loss": 2.8947, + "theoretical_loss": 3.621236535936366, + "tokens_seen": 1084817408 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033905717151454365, + "loss": 2.7032, + "theoretical_loss": 3.621216029539064, + "tokens_seen": 1084882944 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033904714142427283, + "loss": 2.854, + "theoretical_loss": 3.6211955247273107, + "tokens_seen": 1084948480 + }, + { + "epoch": 0.33, + "learning_rate": 0.000339037111334002, + "loss": 2.9125, + "theoretical_loss": 3.6211750215008878, + "tokens_seen": 1085014016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033902708124373124, + "loss": 2.6352, + "theoretical_loss": 3.6211545198595774, + "tokens_seen": 1085079552 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033901705115346037, + "loss": 2.7928, + "theoretical_loss": 3.62113401980316, + "tokens_seen": 1085145088 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003390070210631896, + "loss": 2.8488, + "theoretical_loss": 3.6211135213314183, + "tokens_seen": 1085210624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033899699097291873, + "loss": 2.7162, + "theoretical_loss": 3.621093024444135, + "tokens_seen": 1085276160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033898696088264797, + "loss": 2.9749, + "theoretical_loss": 3.62107252914109, + "tokens_seen": 1085341696 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033897693079237715, + "loss": 2.8548, + "theoretical_loss": 3.621052035422066, + "tokens_seen": 1085407232 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033896690070210633, + "loss": 2.6953, + "theoretical_loss": 3.621031543286845, + "tokens_seen": 1085472768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389568706118355, + "loss": 2.8561, + "theoretical_loss": 3.62101105273521, + "tokens_seen": 1085538304 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389468405215647, + "loss": 2.7889, + "theoretical_loss": 3.6209905637669415, + "tokens_seen": 1085603840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389368104312939, + "loss": 2.7944, + "theoretical_loss": 3.6209700763818224, + "tokens_seen": 1085669376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003389267803410231, + "loss": 2.4984, + "theoretical_loss": 3.6209495905796345, + "tokens_seen": 1085734912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033891675025075224, + "loss": 2.8853, + "theoretical_loss": 3.62092910636016, + "tokens_seen": 1085800448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033890672016048147, + "loss": 2.8857, + "theoretical_loss": 3.6209086237231816, + "tokens_seen": 1085865984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388966900702106, + "loss": 2.8294, + "theoretical_loss": 3.6208881426684805, + "tokens_seen": 1085931520 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033888665997993983, + "loss": 2.6671, + "theoretical_loss": 3.6208676631958396, + "tokens_seen": 1085997056 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338876629889669, + "loss": 2.7934, + "theoretical_loss": 3.6208471853050415, + "tokens_seen": 1086062592 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388665997993982, + "loss": 2.6889, + "theoretical_loss": 3.6208267089958683, + "tokens_seen": 1086128128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388565697091274, + "loss": 2.6472, + "theoretical_loss": 3.6208062342681027, + "tokens_seen": 1086193664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 604042, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7982475757598877, + "objective/train/theoretical_loss": 3.6207857611215264, + "objective/train/tokens_used": 1106719200, + "theoretical_loss": 3.6207857611215264, + "tokens_seen": 1086259200 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388465396188566, + "loss": 2.6263, + "theoretical_loss": 3.6207857611215264, + "tokens_seen": 1086259200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033883650952858574, + "loss": 2.8603, + "theoretical_loss": 3.6207652895559224, + "tokens_seen": 1086324736 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338826479438315, + "loss": 2.5424, + "theoretical_loss": 3.6207448195710734, + "tokens_seen": 1086390272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003388164493480441, + "loss": 2.5165, + "theoretical_loss": 3.620724351166762, + "tokens_seen": 1086455808 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033880641925777334, + "loss": 2.7872, + "theoretical_loss": 3.6207038843427704, + "tokens_seen": 1086521344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387963891675025, + "loss": 2.4773, + "theoretical_loss": 3.6206834190988815, + "tokens_seen": 1086586880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387863590772317, + "loss": 2.7559, + "theoretical_loss": 3.620662955434878, + "tokens_seen": 1086652416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387763289869609, + "loss": 2.975, + "theoretical_loss": 3.6206424933505437, + "tokens_seen": 1086717952 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033876629889669006, + "loss": 2.9142, + "theoretical_loss": 3.6206220328456595, + "tokens_seen": 1086783488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033875626880641924, + "loss": 2.8429, + "theoretical_loss": 3.620601573920009, + "tokens_seen": 1086849024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387462387161485, + "loss": 2.6618, + "theoretical_loss": 3.620581116573376, + "tokens_seen": 1086914560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387362086258776, + "loss": 2.7312, + "theoretical_loss": 3.620560660805542, + "tokens_seen": 1086980096 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033872617853560684, + "loss": 2.5756, + "theoretical_loss": 3.6205402066162913, + "tokens_seen": 1087045632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033871614844533597, + "loss": 2.7202, + "theoretical_loss": 3.6205197540054064, + "tokens_seen": 1087111168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003387061183550652, + "loss": 2.7871, + "theoretical_loss": 3.6204993029726698, + "tokens_seen": 1087176704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386960882647944, + "loss": 2.8353, + "theoretical_loss": 3.6204788535178656, + "tokens_seen": 1087242240 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033868605817452356, + "loss": 2.6144, + "theoretical_loss": 3.620458405640776, + "tokens_seen": 1087307776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033867602808425275, + "loss": 2.8865, + "theoretical_loss": 3.6204379593411846, + "tokens_seen": 1087373312 + }, + { + "epoch": 0.33, + "learning_rate": 0.000338665997993982, + "loss": 2.8391, + "theoretical_loss": 3.6204175146188744, + "tokens_seen": 1087438848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386559679037111, + "loss": 2.7494, + "theoretical_loss": 3.6203970714736298, + "tokens_seen": 1087504384 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033864593781344034, + "loss": 2.806, + "theoretical_loss": 3.6203766299052327, + "tokens_seen": 1087569920 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033863590772316947, + "loss": 2.6098, + "theoretical_loss": 3.6203561899134673, + "tokens_seen": 1087635456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386258776328987, + "loss": 3.0417, + "theoretical_loss": 3.620335751498116, + "tokens_seen": 1087700992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003386158475426279, + "loss": 2.9476, + "theoretical_loss": 3.6203153146589635, + "tokens_seen": 1087766528 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033860581745235707, + "loss": 2.6936, + "theoretical_loss": 3.620294879395793, + "tokens_seen": 1087832064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 604895, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.194870710372925, + "objective/train/theoretical_loss": 3.6202744457083877, + "objective/train/tokens_used": 1108357600, + "theoretical_loss": 3.6202744457083877, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033859578736208625, + "loss": 2.5966, + "theoretical_loss": 3.6202744457083877, + "tokens_seen": 1087897600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033858575727181543, + "loss": 2.8116, + "theoretical_loss": 3.620254013596531, + "tokens_seen": 1087963136 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033857572718154467, + "loss": 2.8302, + "theoretical_loss": 3.620233583060007, + "tokens_seen": 1088028672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033856569709127385, + "loss": 2.77, + "theoretical_loss": 3.6202131540985993, + "tokens_seen": 1088094208 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033855566700100303, + "loss": 2.8116, + "theoretical_loss": 3.6201927267120917, + "tokens_seen": 1088159744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385456369107322, + "loss": 2.9741, + "theoretical_loss": 3.6201723009002675, + "tokens_seen": 1088225280 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033853560682046144, + "loss": 2.8073, + "theoretical_loss": 3.62015187666291, + "tokens_seen": 1088290816 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033852557673019057, + "loss": 2.5589, + "theoretical_loss": 3.6201314539998046, + "tokens_seen": 1088356352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003385155466399198, + "loss": 2.812, + "theoretical_loss": 3.620111032910734, + "tokens_seen": 1088421888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033850551654964893, + "loss": 2.8191, + "theoretical_loss": 3.6200906133954827, + "tokens_seen": 1088487424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033849548645937817, + "loss": 2.9442, + "theoretical_loss": 3.620070195453834, + "tokens_seen": 1088552960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033848545636910735, + "loss": 2.7046, + "theoretical_loss": 3.620049779085573, + "tokens_seen": 1088618496 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033847542627883653, + "loss": 2.8518, + "theoretical_loss": 3.6200293642904824, + "tokens_seen": 1088684032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384653961885657, + "loss": 2.8441, + "theoretical_loss": 3.620008951068347, + "tokens_seen": 1088749568 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384553660982949, + "loss": 2.6895, + "theoretical_loss": 3.6199885394189515, + "tokens_seen": 1088815104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384453360080241, + "loss": 2.9414, + "theoretical_loss": 3.619968129342079, + "tokens_seen": 1088880640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384353059177533, + "loss": 2.9459, + "theoretical_loss": 3.619947720837514, + "tokens_seen": 1088946176 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033842527582748244, + "loss": 2.7135, + "theoretical_loss": 3.619927313905041, + "tokens_seen": 1089011712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033841524573721167, + "loss": 2.7475, + "theoretical_loss": 3.6199069085444444, + "tokens_seen": 1089077248 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003384052156469408, + "loss": 2.5889, + "theoretical_loss": 3.6198865047555087, + "tokens_seen": 1089142784 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033839518555667003, + "loss": 2.7581, + "theoretical_loss": 3.6198661025380177, + "tokens_seen": 1089208320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383851554663992, + "loss": 2.7403, + "theoretical_loss": 3.619845701891756, + "tokens_seen": 1089273856 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383751253761284, + "loss": 2.6924, + "theoretical_loss": 3.6198253028165084, + "tokens_seen": 1089339392 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383650952858576, + "loss": 2.7452, + "theoretical_loss": 3.6198049053120593, + "tokens_seen": 1089404928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383550651955868, + "loss": 2.8973, + "theoretical_loss": 3.6197845093781926, + "tokens_seen": 1089470464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 606118, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.570420980453491, + "objective/train/theoretical_loss": 3.6197641150146938, + "objective/train/tokens_used": 1109996000, + "theoretical_loss": 3.6197641150146938, + "tokens_seen": 1089536000 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033834503510531594, + "loss": 2.6581, + "theoretical_loss": 3.6197641150146938, + "tokens_seen": 1089536000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383350050150452, + "loss": 2.684, + "theoretical_loss": 3.6197437222213473, + "tokens_seen": 1089601536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383249749247743, + "loss": 2.8388, + "theoretical_loss": 3.6197233309979375, + "tokens_seen": 1089667072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033831494483450354, + "loss": 2.7701, + "theoretical_loss": 3.61970294134425, + "tokens_seen": 1089732608 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003383049147442327, + "loss": 2.7718, + "theoretical_loss": 3.619682553260068, + "tokens_seen": 1089798144 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382948846539619, + "loss": 2.6133, + "theoretical_loss": 3.619662166745177, + "tokens_seen": 1089863680 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382848545636911, + "loss": 2.8218, + "theoretical_loss": 3.6196417817993627, + "tokens_seen": 1089929216 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033827482447342026, + "loss": 2.9036, + "theoretical_loss": 3.6196213984224093, + "tokens_seen": 1089994752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033826479438314944, + "loss": 2.937, + "theoretical_loss": 3.619601016614102, + "tokens_seen": 1090060288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382547642928787, + "loss": 2.7739, + "theoretical_loss": 3.6195806363742253, + "tokens_seen": 1090125824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382447342026078, + "loss": 2.7651, + "theoretical_loss": 3.619560257702564, + "tokens_seen": 1090191360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033823470411233704, + "loss": 2.6242, + "theoretical_loss": 3.6195398805989045, + "tokens_seen": 1090256896 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033822467402206617, + "loss": 2.9022, + "theoretical_loss": 3.619519505063031, + "tokens_seen": 1090322432 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382146439317954, + "loss": 2.6241, + "theoretical_loss": 3.619499131094728, + "tokens_seen": 1090387968 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003382046138415246, + "loss": 2.6972, + "theoretical_loss": 3.6194787586937824, + "tokens_seen": 1090453504 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033819458375125376, + "loss": 2.8839, + "theoretical_loss": 3.6194583878599778, + "tokens_seen": 1090519040 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033818455366098295, + "loss": 2.8666, + "theoretical_loss": 3.6194380185931, + "tokens_seen": 1090584576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381745235707122, + "loss": 2.7848, + "theoretical_loss": 3.6194176508929354, + "tokens_seen": 1090650112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381644934804413, + "loss": 2.8002, + "theoretical_loss": 3.619397284759268, + "tokens_seen": 1090715648 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033815446339017054, + "loss": 2.8289, + "theoretical_loss": 3.6193769201918835, + "tokens_seen": 1090781184 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033814443329989967, + "loss": 2.8468, + "theoretical_loss": 3.6193565571905673, + "tokens_seen": 1090846720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381344032096289, + "loss": 2.9158, + "theoretical_loss": 3.619336195755105, + "tokens_seen": 1090912256 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003381243731193581, + "loss": 2.8524, + "theoretical_loss": 3.619315835885283, + "tokens_seen": 1090977792 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033811434302908727, + "loss": 2.6606, + "theoretical_loss": 3.619295477580885, + "tokens_seen": 1091043328 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033810431293881645, + "loss": 3.0112, + "theoretical_loss": 3.6192751208416984, + "tokens_seen": 1091108864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 606665, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6309762001037598, + "objective/train/theoretical_loss": 3.6192547656675083, + "objective/train/tokens_used": 1111634400, + "theoretical_loss": 3.6192547656675083, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033809428284854563, + "loss": 2.6641, + "theoretical_loss": 3.6192547656675083, + "tokens_seen": 1091174400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380842527582748, + "loss": 2.6976, + "theoretical_loss": 3.6192344120581, + "tokens_seen": 1091239936 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033807422266800405, + "loss": 2.7123, + "theoretical_loss": 3.6192140600132596, + "tokens_seen": 1091305472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380641925777332, + "loss": 2.8793, + "theoretical_loss": 3.619193709532773, + "tokens_seen": 1091371008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380541624874624, + "loss": 2.6522, + "theoretical_loss": 3.6191733606164256, + "tokens_seen": 1091436544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033804413239719154, + "loss": 2.8821, + "theoretical_loss": 3.619153013264003, + "tokens_seen": 1091502080 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033803410230692077, + "loss": 2.7992, + "theoretical_loss": 3.6191326674752924, + "tokens_seen": 1091567616 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033802407221664995, + "loss": 2.979, + "theoretical_loss": 3.6191123232500786, + "tokens_seen": 1091633152 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033801404212637913, + "loss": 2.7271, + "theoretical_loss": 3.6190919805881485, + "tokens_seen": 1091698688 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003380040120361083, + "loss": 2.8355, + "theoretical_loss": 3.6190716394892872, + "tokens_seen": 1091764224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033799398194583755, + "loss": 2.7256, + "theoretical_loss": 3.6190512999532807, + "tokens_seen": 1091829760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379839518555667, + "loss": 2.6524, + "theoretical_loss": 3.6190309619799166, + "tokens_seen": 1091895296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379739217652959, + "loss": 2.8708, + "theoretical_loss": 3.619010625568979, + "tokens_seen": 1091960832 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033796389167502504, + "loss": 2.7305, + "theoretical_loss": 3.618990290720256, + "tokens_seen": 1092026368 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379538615847543, + "loss": 2.7191, + "theoretical_loss": 3.618969957433533, + "tokens_seen": 1092091904 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033794383149448346, + "loss": 2.8005, + "theoretical_loss": 3.618949625708596, + "tokens_seen": 1092157440 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033793380140421264, + "loss": 2.7105, + "theoretical_loss": 3.6189292955452315, + "tokens_seen": 1092222976 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379237713139418, + "loss": 2.79, + "theoretical_loss": 3.6189089669432266, + "tokens_seen": 1092288512 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337913741223671, + "loss": 3.0197, + "theoretical_loss": 3.6188886399023668, + "tokens_seen": 1092354048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003379037111334002, + "loss": 2.8093, + "theoretical_loss": 3.618868314422439, + "tokens_seen": 1092419584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378936810431294, + "loss": 2.9259, + "theoretical_loss": 3.61884799050323, + "tokens_seen": 1092485120 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033788365095285854, + "loss": 2.807, + "theoretical_loss": 3.6188276681445255, + "tokens_seen": 1092550656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378736208625878, + "loss": 2.4255, + "theoretical_loss": 3.6188073473461126, + "tokens_seen": 1092616192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378635907723169, + "loss": 2.679, + "theoretical_loss": 3.6187870281077785, + "tokens_seen": 1092681728 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033785356068204614, + "loss": 2.7072, + "theoretical_loss": 3.618766710429308, + "tokens_seen": 1092747264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 607551, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.578968048095703, + "objective/train/theoretical_loss": 3.6187463943104903, + "objective/train/tokens_used": 1113272800, + "theoretical_loss": 3.6187463943104903, + "tokens_seen": 1092812800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378435305917753, + "loss": 2.717, + "theoretical_loss": 3.6187463943104903, + "tokens_seen": 1092812800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378335005015045, + "loss": 2.8585, + "theoretical_loss": 3.6187260797511103, + "tokens_seen": 1092878336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033782347041123374, + "loss": 2.6637, + "theoretical_loss": 3.6187057667509555, + "tokens_seen": 1092943872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378134403209629, + "loss": 2.915, + "theoretical_loss": 3.618685455309813, + "tokens_seen": 1093009408 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003378034102306921, + "loss": 2.9441, + "theoretical_loss": 3.618665145427469, + "tokens_seen": 1093074944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377933801404213, + "loss": 2.9739, + "theoretical_loss": 3.6186448371037105, + "tokens_seen": 1093140480 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033778335005015046, + "loss": 2.8502, + "theoretical_loss": 3.6186245303383253, + "tokens_seen": 1093206016 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033777331995987964, + "loss": 2.7294, + "theoretical_loss": 3.6186042251311, + "tokens_seen": 1093271552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377632898696089, + "loss": 2.8688, + "theoretical_loss": 3.6185839214818207, + "tokens_seen": 1093337088 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337753259779338, + "loss": 3.0609, + "theoretical_loss": 3.618563619390276, + "tokens_seen": 1093402624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033774322968906724, + "loss": 2.6654, + "theoretical_loss": 3.6185433188562515, + "tokens_seen": 1093468160 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033773319959879637, + "loss": 2.6188, + "theoretical_loss": 3.618523019879536, + "tokens_seen": 1093533696 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377231695085256, + "loss": 2.814, + "theoretical_loss": 3.6185027224599153, + "tokens_seen": 1093599232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003377131394182548, + "loss": 2.7198, + "theoretical_loss": 3.6184824265971773, + "tokens_seen": 1093664768 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033770310932798396, + "loss": 2.6649, + "theoretical_loss": 3.6184621322911097, + "tokens_seen": 1093730304 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033769307923771315, + "loss": 2.8431, + "theoretical_loss": 3.618441839541499, + "tokens_seen": 1093795840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376830491474424, + "loss": 2.9257, + "theoretical_loss": 3.618421548348133, + "tokens_seen": 1093861376 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376730190571715, + "loss": 2.6722, + "theoretical_loss": 3.618401258710799, + "tokens_seen": 1093926912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033766298896690074, + "loss": 2.7113, + "theoretical_loss": 3.6183809706292847, + "tokens_seen": 1093992448 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033765295887662987, + "loss": 2.6744, + "theoretical_loss": 3.6183606841033775, + "tokens_seen": 1094057984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376429287863591, + "loss": 2.9785, + "theoretical_loss": 3.618340399132865, + "tokens_seen": 1094123520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003376328986960883, + "loss": 2.7727, + "theoretical_loss": 3.6183201157175344, + "tokens_seen": 1094189056 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033762286860581747, + "loss": 2.9491, + "theoretical_loss": 3.618299833857174, + "tokens_seen": 1094254592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033761283851554665, + "loss": 2.6171, + "theoretical_loss": 3.6182795535515706, + "tokens_seen": 1094320128 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033760280842527583, + "loss": 3.0635, + "theoretical_loss": 3.618259274800513, + "tokens_seen": 1094385664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 607859, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.727546453475952, + "objective/train/theoretical_loss": 3.618238997603788, + "objective/train/tokens_used": 1114911200, + "theoretical_loss": 3.618238997603788, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337592778335005, + "loss": 2.8344, + "theoretical_loss": 3.618238997603788, + "tokens_seen": 1094451200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033758274824473425, + "loss": 2.8925, + "theoretical_loss": 3.618218721961184, + "tokens_seen": 1094516736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375727181544634, + "loss": 2.9069, + "theoretical_loss": 3.618198447872489, + "tokens_seen": 1094582272 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375626880641926, + "loss": 2.8822, + "theoretical_loss": 3.6181781753374898, + "tokens_seen": 1094647808 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033755265797392174, + "loss": 2.8686, + "theoretical_loss": 3.618157904355975, + "tokens_seen": 1094713344 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033754262788365097, + "loss": 2.5076, + "theoretical_loss": 3.6181376349277334, + "tokens_seen": 1094778880 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033753259779338015, + "loss": 2.857, + "theoretical_loss": 3.6181173670525517, + "tokens_seen": 1094844416 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033752256770310933, + "loss": 2.6923, + "theoretical_loss": 3.6180971007302185, + "tokens_seen": 1094909952 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003375125376128385, + "loss": 2.4682, + "theoretical_loss": 3.6180768359605215, + "tokens_seen": 1094975488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033750250752256775, + "loss": 2.9568, + "theoretical_loss": 3.61805657274325, + "tokens_seen": 1095041024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374924774322969, + "loss": 2.7654, + "theoretical_loss": 3.618036311078191, + "tokens_seen": 1095106560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374824473420261, + "loss": 2.8227, + "theoretical_loss": 3.618016050965133, + "tokens_seen": 1095172096 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033747241725175524, + "loss": 2.7536, + "theoretical_loss": 3.617995792403864, + "tokens_seen": 1095237632 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374623871614845, + "loss": 2.5663, + "theoretical_loss": 3.617975535394173, + "tokens_seen": 1095303168 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033745235707121366, + "loss": 2.8911, + "theoretical_loss": 3.617955279935848, + "tokens_seen": 1095368704 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033744232698094284, + "loss": 2.5522, + "theoretical_loss": 3.617935026028677, + "tokens_seen": 1095434240 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337432296890672, + "loss": 2.6732, + "theoretical_loss": 3.6179147736724495, + "tokens_seen": 1095499776 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374222668004012, + "loss": 2.9258, + "theoretical_loss": 3.617894522866953, + "tokens_seen": 1095565312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374122367101304, + "loss": 2.6745, + "theoretical_loss": 3.6178742736119758, + "tokens_seen": 1095630848 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003374022066198596, + "loss": 2.4884, + "theoretical_loss": 3.617854025907307, + "tokens_seen": 1095696384 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033739217652958874, + "loss": 2.5483, + "theoretical_loss": 3.6178337797527353, + "tokens_seen": 1095761920 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337382146439318, + "loss": 2.7345, + "theoretical_loss": 3.6178135351480494, + "tokens_seen": 1095827456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373721163490471, + "loss": 2.825, + "theoretical_loss": 3.617793292093037, + "tokens_seen": 1095892992 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033736208625877634, + "loss": 2.8887, + "theoretical_loss": 3.6177730505874885, + "tokens_seen": 1095958528 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373520561685055, + "loss": 2.8044, + "theoretical_loss": 3.617752810631191, + "tokens_seen": 1096024064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 608889, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.800088882446289, + "objective/train/theoretical_loss": 3.6177325722239337, + "objective/train/tokens_used": 1116549600, + "theoretical_loss": 3.6177325722239337, + "tokens_seen": 1096089600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373420260782347, + "loss": 2.8135, + "theoretical_loss": 3.6177325722239337, + "tokens_seen": 1096089600 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373319959879639, + "loss": 2.7077, + "theoretical_loss": 3.617712335365506, + "tokens_seen": 1096155136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373219658976931, + "loss": 2.7595, + "theoretical_loss": 3.6176921000556965, + "tokens_seen": 1096220672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033731193580742225, + "loss": 2.9444, + "theoretical_loss": 3.617671866294294, + "tokens_seen": 1096286208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003373019057171515, + "loss": 2.6648, + "theoretical_loss": 3.6176516340810876, + "tokens_seen": 1096351744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372918756268806, + "loss": 2.7699, + "theoretical_loss": 3.6176314034158663, + "tokens_seen": 1096417280 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033728184553660984, + "loss": 2.8946, + "theoretical_loss": 3.6176111742984194, + "tokens_seen": 1096482816 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337271815446339, + "loss": 2.7141, + "theoretical_loss": 3.6175909467285354, + "tokens_seen": 1096548352 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372617853560682, + "loss": 2.8175, + "theoretical_loss": 3.617570720706004, + "tokens_seen": 1096613888 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372517552657974, + "loss": 2.9539, + "theoretical_loss": 3.6175504962306135, + "tokens_seen": 1096679424 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033724172517552657, + "loss": 2.7993, + "theoretical_loss": 3.6175302733021546, + "tokens_seen": 1096744960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033723169508525575, + "loss": 2.8731, + "theoretical_loss": 3.617510051920415, + "tokens_seen": 1096810496 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337221664994985, + "loss": 2.6325, + "theoretical_loss": 3.6174898320851847, + "tokens_seen": 1096876032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003372116349047141, + "loss": 2.826, + "theoretical_loss": 3.617469613796253, + "tokens_seen": 1096941568 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033720160481444335, + "loss": 2.8224, + "theoretical_loss": 3.6174493970534094, + "tokens_seen": 1097007104 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371915747241725, + "loss": 2.8515, + "theoretical_loss": 3.6174291818564432, + "tokens_seen": 1097072640 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371815446339017, + "loss": 2.7638, + "theoretical_loss": 3.617408968205144, + "tokens_seen": 1097138176 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371715145436309, + "loss": 2.8324, + "theoretical_loss": 3.6173887560993006, + "tokens_seen": 1097203712 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033716148445336007, + "loss": 2.891, + "theoretical_loss": 3.6173685455387035, + "tokens_seen": 1097269248 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033715145436308925, + "loss": 2.8952, + "theoretical_loss": 3.6173483365231416, + "tokens_seen": 1097334784 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371414242728185, + "loss": 2.7169, + "theoretical_loss": 3.6173281290524053, + "tokens_seen": 1097400320 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371313941825476, + "loss": 2.8957, + "theoretical_loss": 3.6173079231262832, + "tokens_seen": 1097465856 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033712136409227685, + "loss": 2.7076, + "theoretical_loss": 3.6172877187445662, + "tokens_seen": 1097531392 + }, + { + "epoch": 0.33, + "learning_rate": 0.000337111334002006, + "loss": 2.7603, + "theoretical_loss": 3.6172675159070424, + "tokens_seen": 1097596928 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003371013039117352, + "loss": 2.7749, + "theoretical_loss": 3.617247314613503, + "tokens_seen": 1097662464 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 610057, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0604283809661865, + "objective/train/theoretical_loss": 3.617227114863738, + "objective/train/tokens_used": 1118188000, + "theoretical_loss": 3.617227114863738, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370912738214644, + "loss": 3.0146, + "theoretical_loss": 3.617227114863738, + "tokens_seen": 1097728000 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370812437311936, + "loss": 2.8418, + "theoretical_loss": 3.617206916657536, + "tokens_seen": 1097793536 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370712136409228, + "loss": 2.7766, + "theoretical_loss": 3.617186719994688, + "tokens_seen": 1097859072 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033706118355065194, + "loss": 2.7925, + "theoretical_loss": 3.6171665248749836, + "tokens_seen": 1097924608 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033705115346038117, + "loss": 2.6988, + "theoretical_loss": 3.6171463312982124, + "tokens_seen": 1097990144 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033704112337011035, + "loss": 2.9721, + "theoretical_loss": 3.617126139264165, + "tokens_seen": 1098055680 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033703109327983953, + "loss": 2.8138, + "theoretical_loss": 3.617105948772631, + "tokens_seen": 1098121216 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370210631895687, + "loss": 2.5797, + "theoretical_loss": 3.617085759823401, + "tokens_seen": 1098186752 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033701103309929795, + "loss": 2.7429, + "theoretical_loss": 3.6170655724162653, + "tokens_seen": 1098252288 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003370010030090271, + "loss": 2.6726, + "theoretical_loss": 3.6170453865510135, + "tokens_seen": 1098317824 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369909729187563, + "loss": 2.8859, + "theoretical_loss": 3.6170252022274356, + "tokens_seen": 1098383360 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033698094282848544, + "loss": 2.7337, + "theoretical_loss": 3.6170050194453225, + "tokens_seen": 1098448896 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369709127382147, + "loss": 2.7052, + "theoretical_loss": 3.616984838204465, + "tokens_seen": 1098514432 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033696088264794386, + "loss": 2.6465, + "theoretical_loss": 3.616964658504652, + "tokens_seen": 1098579968 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033695085255767304, + "loss": 2.5825, + "theoretical_loss": 3.6169444803456754, + "tokens_seen": 1098645504 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369408224674022, + "loss": 2.7459, + "theoretical_loss": 3.616924303727325, + "tokens_seen": 1098711040 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369307923771314, + "loss": 2.722, + "theoretical_loss": 3.616904128649391, + "tokens_seen": 1098776576 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369207622868606, + "loss": 2.6449, + "theoretical_loss": 3.616883955111664, + "tokens_seen": 1098842112 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003369107321965898, + "loss": 2.7865, + "theoretical_loss": 3.616863783113935, + "tokens_seen": 1098907648 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033690070210631894, + "loss": 2.7845, + "theoretical_loss": 3.6168436126559946, + "tokens_seen": 1098973184 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368906720160482, + "loss": 2.5238, + "theoretical_loss": 3.616823443737633, + "tokens_seen": 1099038720 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368806419257773, + "loss": 2.7368, + "theoretical_loss": 3.616803276358641, + "tokens_seen": 1099104256 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033687061183550654, + "loss": 2.6779, + "theoretical_loss": 3.61678311051881, + "tokens_seen": 1099169792 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368605817452357, + "loss": 2.7011, + "theoretical_loss": 3.61676294621793, + "tokens_seen": 1099235328 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368505516549649, + "loss": 2.7203, + "theoretical_loss": 3.6167427834557913, + "tokens_seen": 1099300864 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 610651, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6474804878234863, + "objective/train/theoretical_loss": 3.6167226222321864, + "objective/train/tokens_used": 1119826400, + "theoretical_loss": 3.6167226222321864, + "tokens_seen": 1099366400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368405215646941, + "loss": 2.5971, + "theoretical_loss": 3.6167226222321864, + "tokens_seen": 1099366400 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368304914744233, + "loss": 2.7681, + "theoretical_loss": 3.616702462546905, + "tokens_seen": 1099431936 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033682046138415245, + "loss": 2.737, + "theoretical_loss": 3.6166823043997383, + "tokens_seen": 1099497472 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368104312938817, + "loss": 2.7658, + "theoretical_loss": 3.6166621477904775, + "tokens_seen": 1099563008 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003368004012036108, + "loss": 2.4798, + "theoretical_loss": 3.6166419927189133, + "tokens_seen": 1099628544 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033679037111334004, + "loss": 2.4484, + "theoretical_loss": 3.6166218391848366, + "tokens_seen": 1099694080 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367803410230692, + "loss": 2.7835, + "theoretical_loss": 3.616601687188039, + "tokens_seen": 1099759616 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367703109327984, + "loss": 2.6162, + "theoretical_loss": 3.6165815367283116, + "tokens_seen": 1099825152 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367602808425276, + "loss": 2.8223, + "theoretical_loss": 3.6165613878054454, + "tokens_seen": 1099890688 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033675025075225677, + "loss": 2.8348, + "theoretical_loss": 3.616541240419232, + "tokens_seen": 1099956224 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033674022066198595, + "loss": 2.9311, + "theoretical_loss": 3.6165210945694612, + "tokens_seen": 1100021760 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367301905717152, + "loss": 2.9308, + "theoretical_loss": 3.616500950255926, + "tokens_seen": 1100087296 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003367201604814443, + "loss": 2.5941, + "theoretical_loss": 3.6164808074784176, + "tokens_seen": 1100152832 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033671013039117355, + "loss": 2.7612, + "theoretical_loss": 3.6164606662367262, + "tokens_seen": 1100218368 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033670010030090273, + "loss": 2.7321, + "theoretical_loss": 3.6164405265306447, + "tokens_seen": 1100283904 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366900702106319, + "loss": 2.6555, + "theoretical_loss": 3.6164203883599635, + "tokens_seen": 1100349440 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366800401203611, + "loss": 2.7248, + "theoretical_loss": 3.6164002517244738, + "tokens_seen": 1100414976 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033667001003009027, + "loss": 2.9274, + "theoretical_loss": 3.6163801166239686, + "tokens_seen": 1100480512 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033665997993981945, + "loss": 2.6037, + "theoretical_loss": 3.6163599830582385, + "tokens_seen": 1100546048 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366499498495487, + "loss": 2.6833, + "theoretical_loss": 3.6163398510270754, + "tokens_seen": 1100611584 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366399197592778, + "loss": 2.9599, + "theoretical_loss": 3.6163197205302704, + "tokens_seen": 1100677120 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033662988966900705, + "loss": 2.6235, + "theoretical_loss": 3.616299591567616, + "tokens_seen": 1100742656 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366198595787362, + "loss": 2.6697, + "theoretical_loss": 3.6162794641389038, + "tokens_seen": 1100808192 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003366098294884654, + "loss": 2.764, + "theoretical_loss": 3.616259338243925, + "tokens_seen": 1100873728 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365997993981946, + "loss": 2.7095, + "theoretical_loss": 3.6162392138824724, + "tokens_seen": 1100939264 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 611659, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3433709144592285, + "objective/train/theoretical_loss": 3.6162190910543366, + "objective/train/tokens_used": 1121464800, + "theoretical_loss": 3.6162190910543366, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365897693079238, + "loss": 2.5755, + "theoretical_loss": 3.6162190910543366, + "tokens_seen": 1101004800 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033657973921765295, + "loss": 2.8004, + "theoretical_loss": 3.616198969759311, + "tokens_seen": 1101070336 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033656970912738214, + "loss": 2.7259, + "theoretical_loss": 3.6161788499971865, + "tokens_seen": 1101135872 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365596790371113, + "loss": 2.8029, + "theoretical_loss": 3.6161587317677553, + "tokens_seen": 1101201408 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033654964894684055, + "loss": 2.6944, + "theoretical_loss": 3.6161386150708097, + "tokens_seen": 1101266944 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365396188565697, + "loss": 2.7386, + "theoretical_loss": 3.6161184999061415, + "tokens_seen": 1101332480 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365295887662989, + "loss": 2.6169, + "theoretical_loss": 3.616098386273543, + "tokens_seen": 1101398016 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365195586760281, + "loss": 2.9178, + "theoretical_loss": 3.6160782741728066, + "tokens_seen": 1101463552 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003365095285857573, + "loss": 2.8122, + "theoretical_loss": 3.616058163603724, + "tokens_seen": 1101529088 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033649949849548646, + "loss": 2.6384, + "theoretical_loss": 3.6160380545660873, + "tokens_seen": 1101594624 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033648946840521564, + "loss": 2.9066, + "theoretical_loss": 3.6160179470596896, + "tokens_seen": 1101660160 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364794383149448, + "loss": 2.8531, + "theoretical_loss": 3.6159978410843223, + "tokens_seen": 1101725696 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033646940822467406, + "loss": 2.5062, + "theoretical_loss": 3.6159777366397785, + "tokens_seen": 1101791232 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364593781344032, + "loss": 2.7402, + "theoretical_loss": 3.6159576337258503, + "tokens_seen": 1101856768 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364493480441324, + "loss": 2.8608, + "theoretical_loss": 3.61593753234233, + "tokens_seen": 1101922304 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033643931795386154, + "loss": 2.7708, + "theoretical_loss": 3.6159174324890104, + "tokens_seen": 1101987840 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003364292878635908, + "loss": 3.0905, + "theoretical_loss": 3.6158973341656835, + "tokens_seen": 1102053376 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033641925777331996, + "loss": 2.9452, + "theoretical_loss": 3.6158772373721426, + "tokens_seen": 1102118912 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033640922768304914, + "loss": 2.8754, + "theoretical_loss": 3.6158571421081795, + "tokens_seen": 1102184448 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363991975927783, + "loss": 2.7918, + "theoretical_loss": 3.6158370483735878, + "tokens_seen": 1102249984 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363891675025075, + "loss": 3.0109, + "theoretical_loss": 3.6158169561681595, + "tokens_seen": 1102315520 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363791374122367, + "loss": 2.9653, + "theoretical_loss": 3.6157968654916868, + "tokens_seen": 1102381056 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363691073219659, + "loss": 2.9488, + "theoretical_loss": 3.6157767763439637, + "tokens_seen": 1102446592 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033635907723169505, + "loss": 2.751, + "theoretical_loss": 3.6157566887247823, + "tokens_seen": 1102512128 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363490471414243, + "loss": 2.9518, + "theoretical_loss": 3.6157366026339357, + "tokens_seen": 1102577664 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 612217, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9327733516693115, + "objective/train/theoretical_loss": 3.615716518071217, + "objective/train/tokens_used": 1123103200, + "theoretical_loss": 3.615716518071217, + "tokens_seen": 1102643200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033633901705115346, + "loss": 2.9102, + "theoretical_loss": 3.615716518071217, + "tokens_seen": 1102643200 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033632898696088265, + "loss": 2.7885, + "theoretical_loss": 3.6156964350364182, + "tokens_seen": 1102708736 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003363189568706119, + "loss": 2.7765, + "theoretical_loss": 3.615676353529333, + "tokens_seen": 1102774272 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336308926780341, + "loss": 2.9011, + "theoretical_loss": 3.6156562735497544, + "tokens_seen": 1102839808 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033629889669007024, + "loss": 2.8588, + "theoretical_loss": 3.6156361950974754, + "tokens_seen": 1102905344 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362888665997994, + "loss": 2.6921, + "theoretical_loss": 3.615616118172289, + "tokens_seen": 1102970880 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362788365095286, + "loss": 2.725, + "theoretical_loss": 3.615596042773988, + "tokens_seen": 1103036416 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362688064192578, + "loss": 2.8121, + "theoretical_loss": 3.615575968902366, + "tokens_seen": 1103101952 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033625877632898697, + "loss": 2.745, + "theoretical_loss": 3.6155558965572165, + "tokens_seen": 1103167488 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033624874623871615, + "loss": 2.7481, + "theoretical_loss": 3.6155358257383323, + "tokens_seen": 1103233024 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362387161484454, + "loss": 2.7961, + "theoretical_loss": 3.6155157564455065, + "tokens_seen": 1103298560 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003362286860581745, + "loss": 2.536, + "theoretical_loss": 3.615495688678533, + "tokens_seen": 1103364096 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033621865596790375, + "loss": 2.5681, + "theoretical_loss": 3.6154756224372044, + "tokens_seen": 1103429632 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033620862587763293, + "loss": 2.7488, + "theoretical_loss": 3.615455557721315, + "tokens_seen": 1103495168 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361985957873621, + "loss": 2.7733, + "theoretical_loss": 3.615435494530658, + "tokens_seen": 1103560704 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361885656970913, + "loss": 2.7637, + "theoretical_loss": 3.615415432865026, + "tokens_seen": 1103626240 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033617853560682047, + "loss": 2.5859, + "theoretical_loss": 3.6153953727242136, + "tokens_seen": 1103691776 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033616850551654965, + "loss": 3.0173, + "theoretical_loss": 3.6153753141080145, + "tokens_seen": 1103757312 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361584754262789, + "loss": 2.6315, + "theoretical_loss": 3.615355257016221, + "tokens_seen": 1103822848 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336148445336008, + "loss": 2.6973, + "theoretical_loss": 3.615335201448628, + "tokens_seen": 1103888384 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033613841524573725, + "loss": 2.8397, + "theoretical_loss": 3.6153151474050285, + "tokens_seen": 1103953920 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361283851554664, + "loss": 3.0104, + "theoretical_loss": 3.6152950948852167, + "tokens_seen": 1104019456 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361183550651956, + "loss": 2.8473, + "theoretical_loss": 3.6152750438889862, + "tokens_seen": 1104084992 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003361083249749248, + "loss": 2.7402, + "theoretical_loss": 3.6152549944161314, + "tokens_seen": 1104150528 + }, + { + "epoch": 0.33, + "learning_rate": 0.000336098294884654, + "loss": 2.5521, + "theoretical_loss": 3.615234946466445, + "tokens_seen": 1104216064 + }, + { + "epoch": 0.33, + "objective/train/docs_used": 613498, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7340283393859863, + "objective/train/theoretical_loss": 3.615214900039721, + "objective/train/tokens_used": 1124741600, + "theoretical_loss": 3.615214900039721, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033608826479438316, + "loss": 2.6072, + "theoretical_loss": 3.615214900039721, + "tokens_seen": 1104281600 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033607823470411234, + "loss": 2.713, + "theoretical_loss": 3.6151948551357544, + "tokens_seen": 1104347136 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360682046138415, + "loss": 2.7713, + "theoretical_loss": 3.6151748117543385, + "tokens_seen": 1104412672 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033605817452357075, + "loss": 2.7701, + "theoretical_loss": 3.615154769895267, + "tokens_seen": 1104478208 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360481444332999, + "loss": 2.7249, + "theoretical_loss": 3.6151347295583345, + "tokens_seen": 1104543744 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360381143430291, + "loss": 2.7956, + "theoretical_loss": 3.615114690743335, + "tokens_seen": 1104609280 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360280842527583, + "loss": 2.9324, + "theoretical_loss": 3.6150946534500625, + "tokens_seen": 1104674816 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003360180541624875, + "loss": 2.7668, + "theoretical_loss": 3.615074617678311, + "tokens_seen": 1104740352 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033600802407221666, + "loss": 2.775, + "theoretical_loss": 3.6150545834278756, + "tokens_seen": 1104805888 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033599799398194584, + "loss": 2.7627, + "theoretical_loss": 3.615034550698549, + "tokens_seen": 1104871424 + }, + { + "epoch": 0.33, + "learning_rate": 0.000335987963891675, + "loss": 2.7189, + "theoretical_loss": 3.615014519490127, + "tokens_seen": 1104936960 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033597793380140426, + "loss": 2.8479, + "theoretical_loss": 3.6149944898024033, + "tokens_seen": 1105002496 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359679037111334, + "loss": 2.7308, + "theoretical_loss": 3.614974461635172, + "tokens_seen": 1105068032 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359578736208626, + "loss": 2.742, + "theoretical_loss": 3.6149544349882277, + "tokens_seen": 1105133568 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033594784353059174, + "loss": 2.8406, + "theoretical_loss": 3.614934409861365, + "tokens_seen": 1105199104 + }, + { + "epoch": 0.33, + "learning_rate": 0.000335937813440321, + "loss": 2.8199, + "theoretical_loss": 3.6149143862543784, + "tokens_seen": 1105264640 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033592778335005016, + "loss": 2.737, + "theoretical_loss": 3.614894364167063, + "tokens_seen": 1105330176 + }, + { + "epoch": 0.33, + "learning_rate": 0.00033591775325977934, + "loss": 2.8659, + "theoretical_loss": 3.614874343599212, + "tokens_seen": 1105395712 + }, + { + "epoch": 0.33, + "learning_rate": 0.0003359077231695085, + "loss": 2.751, + "theoretical_loss": 3.614854324550621, + "tokens_seen": 1105461248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358976930792377, + "loss": 2.978, + "theoretical_loss": 3.614834307021084, + "tokens_seen": 1105526784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358876629889669, + "loss": 2.6978, + "theoretical_loss": 3.614814291010397, + "tokens_seen": 1105592320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358776328986961, + "loss": 2.9026, + "theoretical_loss": 3.6147942765183534, + "tokens_seen": 1105657856 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033586760280842525, + "loss": 2.7393, + "theoretical_loss": 3.6147742635447484, + "tokens_seen": 1105723392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358575727181545, + "loss": 2.8325, + "theoretical_loss": 3.614754252089377, + "tokens_seen": 1105788928 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033584754262788366, + "loss": 3.0232, + "theoretical_loss": 3.614734242152034, + "tokens_seen": 1105854464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 614567, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4206838607788086, + "objective/train/theoretical_loss": 3.6147142337325135, + "objective/train/tokens_used": 1126380000, + "theoretical_loss": 3.6147142337325135, + "tokens_seen": 1105920000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033583751253761285, + "loss": 2.6926, + "theoretical_loss": 3.6147142337325135, + "tokens_seen": 1105920000 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335827482447342, + "loss": 2.8486, + "theoretical_loss": 3.6146942268306117, + "tokens_seen": 1105985536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358174523570712, + "loss": 2.7974, + "theoretical_loss": 3.6146742214461227, + "tokens_seen": 1106051072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003358074222668004, + "loss": 2.7092, + "theoretical_loss": 3.614654217578842, + "tokens_seen": 1106116608 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357973921765296, + "loss": 2.5362, + "theoretical_loss": 3.6146342152285644, + "tokens_seen": 1106182144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033578736208625875, + "loss": 2.8508, + "theoretical_loss": 3.614614214395085, + "tokens_seen": 1106247680 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335777331995988, + "loss": 2.4857, + "theoretical_loss": 3.614594215078199, + "tokens_seen": 1106313216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357673019057171, + "loss": 2.8308, + "theoretical_loss": 3.614574217277702, + "tokens_seen": 1106378752 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033575727181544635, + "loss": 2.8693, + "theoretical_loss": 3.614554220993388, + "tokens_seen": 1106444288 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033574724172517553, + "loss": 2.7059, + "theoretical_loss": 3.6145342262250533, + "tokens_seen": 1106509824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357372116349047, + "loss": 2.7467, + "theoretical_loss": 3.6145142329724926, + "tokens_seen": 1106575360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003357271815446339, + "loss": 2.6414, + "theoretical_loss": 3.6144942412355023, + "tokens_seen": 1106640896 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033571715145436313, + "loss": 2.8767, + "theoretical_loss": 3.6144742510138768, + "tokens_seen": 1106706432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033570712136409225, + "loss": 2.8899, + "theoretical_loss": 3.614454262307411, + "tokens_seen": 1106771968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356970912738215, + "loss": 2.8324, + "theoretical_loss": 3.6144342751159018, + "tokens_seen": 1106837504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356870611835506, + "loss": 2.5821, + "theoretical_loss": 3.614414289439144, + "tokens_seen": 1106903040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033567703109327985, + "loss": 2.6711, + "theoretical_loss": 3.6143943052769325, + "tokens_seen": 1106968576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033566700100300903, + "loss": 2.7409, + "theoretical_loss": 3.6143743226290637, + "tokens_seen": 1107034112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356569709127382, + "loss": 2.8463, + "theoretical_loss": 3.614354341495333, + "tokens_seen": 1107099648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356469408224674, + "loss": 2.7817, + "theoretical_loss": 3.614334361875536, + "tokens_seen": 1107165184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356369107321966, + "loss": 2.7057, + "theoretical_loss": 3.614314383769469, + "tokens_seen": 1107230720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033562688064192576, + "loss": 2.9353, + "theoretical_loss": 3.6142944071769265, + "tokens_seen": 1107296256 + }, + { + "epoch": 0.34, + "learning_rate": 0.000335616850551655, + "loss": 2.4972, + "theoretical_loss": 3.614274432097705, + "tokens_seen": 1107361792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003356068204613841, + "loss": 2.7738, + "theoretical_loss": 3.6142544585316, + "tokens_seen": 1107427328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033559679037111336, + "loss": 2.7836, + "theoretical_loss": 3.614234486478408, + "tokens_seen": 1107492864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 615343, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.606733798980713, + "objective/train/theoretical_loss": 3.614214515937924, + "objective/train/tokens_used": 1128018400, + "theoretical_loss": 3.614214515937924, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355867602808425, + "loss": 2.9232, + "theoretical_loss": 3.614214515937924, + "tokens_seen": 1107558400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355767301905717, + "loss": 2.8684, + "theoretical_loss": 3.614194546909945, + "tokens_seen": 1107623936 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033556670010030095, + "loss": 2.6234, + "theoretical_loss": 3.6141745793942657, + "tokens_seen": 1107689472 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355566700100301, + "loss": 2.6494, + "theoretical_loss": 3.6141546133906832, + "tokens_seen": 1107755008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355466399197593, + "loss": 2.9153, + "theoretical_loss": 3.6141346488989923, + "tokens_seen": 1107820544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355366098294885, + "loss": 2.7833, + "theoretical_loss": 3.6141146859189908, + "tokens_seen": 1107886080 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003355265797392177, + "loss": 2.5932, + "theoretical_loss": 3.6140947244504735, + "tokens_seen": 1107951616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033551654964894686, + "loss": 2.6819, + "theoretical_loss": 3.614074764493237, + "tokens_seen": 1108017152 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033550651955867604, + "loss": 2.9393, + "theoretical_loss": 3.6140548060470774, + "tokens_seen": 1108082688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354964894684052, + "loss": 3.0045, + "theoretical_loss": 3.614034849111791, + "tokens_seen": 1108148224 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033548645937813446, + "loss": 2.7504, + "theoretical_loss": 3.614014893687174, + "tokens_seen": 1108213760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354764292878636, + "loss": 2.8945, + "theoretical_loss": 3.613994939773023, + "tokens_seen": 1108279296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354663991975928, + "loss": 2.7427, + "theoretical_loss": 3.613974987369134, + "tokens_seen": 1108344832 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033545636910732195, + "loss": 2.9324, + "theoretical_loss": 3.613955036475304, + "tokens_seen": 1108410368 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354463390170512, + "loss": 2.575, + "theoretical_loss": 3.6139350870913285, + "tokens_seen": 1108475904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033543630892678036, + "loss": 2.6235, + "theoretical_loss": 3.6139151392170046, + "tokens_seen": 1108541440 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033542627883650954, + "loss": 2.7408, + "theoretical_loss": 3.613895192852129, + "tokens_seen": 1108606976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354162487462387, + "loss": 2.7964, + "theoretical_loss": 3.6138752479964973, + "tokens_seen": 1108672512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003354062186559679, + "loss": 2.8134, + "theoretical_loss": 3.6138553046499076, + "tokens_seen": 1108738048 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353961885656971, + "loss": 2.683, + "theoretical_loss": 3.6138353628121553, + "tokens_seen": 1108803584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353861584754263, + "loss": 2.4956, + "theoretical_loss": 3.6138154224830377, + "tokens_seen": 1108869120 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033537612838515545, + "loss": 2.766, + "theoretical_loss": 3.6137954836623507, + "tokens_seen": 1108934656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353660982948847, + "loss": 2.6687, + "theoretical_loss": 3.613775546349892, + "tokens_seen": 1109000192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033535606820461386, + "loss": 2.9337, + "theoretical_loss": 3.613755610545458, + "tokens_seen": 1109065728 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033534603811434305, + "loss": 2.6042, + "theoretical_loss": 3.6137356762488464, + "tokens_seen": 1109131264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 616276, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.3585472106933594, + "objective/train/theoretical_loss": 3.613715743459852, + "objective/train/tokens_used": 1129656800, + "theoretical_loss": 3.613715743459852, + "tokens_seen": 1109196800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353360080240722, + "loss": 3.0601, + "theoretical_loss": 3.613715743459852, + "tokens_seen": 1109196800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353259779338014, + "loss": 2.673, + "theoretical_loss": 3.613695812178274, + "tokens_seen": 1109262336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353159478435306, + "loss": 2.7332, + "theoretical_loss": 3.613675882403908, + "tokens_seen": 1109327872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003353059177532598, + "loss": 2.997, + "theoretical_loss": 3.6136559541365507, + "tokens_seen": 1109393408 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033529588766298895, + "loss": 2.732, + "theoretical_loss": 3.6136360273760006, + "tokens_seen": 1109458944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352858575727182, + "loss": 2.7252, + "theoretical_loss": 3.6136161021220534, + "tokens_seen": 1109524480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352758274824473, + "loss": 2.8383, + "theoretical_loss": 3.6135961783745065, + "tokens_seen": 1109590016 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033526579739217655, + "loss": 2.836, + "theoretical_loss": 3.613576256133158, + "tokens_seen": 1109655552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033525576730190573, + "loss": 2.694, + "theoretical_loss": 3.6135563353978037, + "tokens_seen": 1109721088 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352457372116349, + "loss": 2.6563, + "theoretical_loss": 3.613536416168242, + "tokens_seen": 1109786624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352357071213641, + "loss": 2.497, + "theoretical_loss": 3.6135164984442696, + "tokens_seen": 1109852160 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033522567703109333, + "loss": 2.6525, + "theoretical_loss": 3.6134965822256837, + "tokens_seen": 1109917696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033521564694082245, + "loss": 2.7866, + "theoretical_loss": 3.613476667512282, + "tokens_seen": 1109983232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003352056168505517, + "loss": 2.7501, + "theoretical_loss": 3.6134567543038614, + "tokens_seen": 1110048768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351955867602808, + "loss": 2.7559, + "theoretical_loss": 3.61343684260022, + "tokens_seen": 1110114304 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033518555667001005, + "loss": 2.8027, + "theoretical_loss": 3.6134169324011545, + "tokens_seen": 1110179840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033517552657973923, + "loss": 2.7998, + "theoretical_loss": 3.613397023706463, + "tokens_seen": 1110245376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351654964894684, + "loss": 2.9335, + "theoretical_loss": 3.6133771165159425, + "tokens_seen": 1110310912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351554663991976, + "loss": 2.8296, + "theoretical_loss": 3.6133572108293914, + "tokens_seen": 1110376448 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351454363089268, + "loss": 2.7763, + "theoretical_loss": 3.613337306646607, + "tokens_seen": 1110441984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033513540621865596, + "loss": 2.7825, + "theoretical_loss": 3.613317403967386, + "tokens_seen": 1110507520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351253761283852, + "loss": 3.0032, + "theoretical_loss": 3.6132975027915277, + "tokens_seen": 1110573056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003351153460381143, + "loss": 2.4999, + "theoretical_loss": 3.6132776031188287, + "tokens_seen": 1110638592 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033510531594784356, + "loss": 2.7841, + "theoretical_loss": 3.6132577049490866, + "tokens_seen": 1110704128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350952858575727, + "loss": 2.7022, + "theoretical_loss": 3.6132378082821006, + "tokens_seen": 1110769664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 616833, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7672462463378906, + "objective/train/theoretical_loss": 3.613217913117667, + "objective/train/tokens_used": 1131295200, + "theoretical_loss": 3.613217913117667, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350852557673019, + "loss": 2.8862, + "theoretical_loss": 3.613217913117667, + "tokens_seen": 1110835200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350752256770311, + "loss": 3.0087, + "theoretical_loss": 3.6131980194555844, + "tokens_seen": 1110900736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350651955867603, + "loss": 2.8562, + "theoretical_loss": 3.6131781272956514, + "tokens_seen": 1110966272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033505516549648946, + "loss": 2.8075, + "theoretical_loss": 3.6131582366376644, + "tokens_seen": 1111031808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350451354062187, + "loss": 2.7964, + "theoretical_loss": 3.613138347481423, + "tokens_seen": 1111097344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350351053159478, + "loss": 2.6795, + "theoretical_loss": 3.6131184598267243, + "tokens_seen": 1111162880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033502507522567706, + "loss": 2.5942, + "theoretical_loss": 3.6130985736733665, + "tokens_seen": 1111228416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350150451354062, + "loss": 2.6254, + "theoretical_loss": 3.613078689021148, + "tokens_seen": 1111293952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003350050150451354, + "loss": 2.944, + "theoretical_loss": 3.6130588058698665, + "tokens_seen": 1111359488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349949849548646, + "loss": 2.7659, + "theoretical_loss": 3.613038924219321, + "tokens_seen": 1111425024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349849548645938, + "loss": 2.7844, + "theoretical_loss": 3.6130190440693095, + "tokens_seen": 1111490560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033497492477432296, + "loss": 2.8088, + "theoretical_loss": 3.6129991654196294, + "tokens_seen": 1111556096 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033496489468405215, + "loss": 2.7411, + "theoretical_loss": 3.61297928827008, + "tokens_seen": 1111621632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349548645937813, + "loss": 2.8393, + "theoretical_loss": 3.612959412620459, + "tokens_seen": 1111687168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033494483450351056, + "loss": 2.7167, + "theoretical_loss": 3.612939538470566, + "tokens_seen": 1111752704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349348044132397, + "loss": 2.8403, + "theoretical_loss": 3.612919665820198, + "tokens_seen": 1111818240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349247743229689, + "loss": 2.7531, + "theoretical_loss": 3.612899794669154, + "tokens_seen": 1111883776 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033491474423269805, + "loss": 2.5888, + "theoretical_loss": 3.6128799250172325, + "tokens_seen": 1111949312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003349047141424273, + "loss": 2.6246, + "theoretical_loss": 3.6128600568642324, + "tokens_seen": 1112014848 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033489468405215647, + "loss": 2.6785, + "theoretical_loss": 3.612840190209952, + "tokens_seen": 1112080384 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033488465396188565, + "loss": 2.512, + "theoretical_loss": 3.61282032505419, + "tokens_seen": 1112145920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033487462387161483, + "loss": 2.6301, + "theoretical_loss": 3.612800461396745, + "tokens_seen": 1112211456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033486459378134406, + "loss": 2.7051, + "theoretical_loss": 3.612780599237416, + "tokens_seen": 1112276992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348545636910732, + "loss": 2.7752, + "theoretical_loss": 3.612760738576001, + "tokens_seen": 1112342528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348445336008024, + "loss": 2.7236, + "theoretical_loss": 3.6127408794123, + "tokens_seen": 1112408064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 617863, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.2659218311309814, + "objective/train/theoretical_loss": 3.6127210217461108, + "objective/train/tokens_used": 1132933600, + "theoretical_loss": 3.6127210217461108, + "tokens_seen": 1112473600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348345035105316, + "loss": 2.6578, + "theoretical_loss": 3.6127210217461108, + "tokens_seen": 1112473600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003348244734202608, + "loss": 3.0378, + "theoretical_loss": 3.6127011655772323, + "tokens_seen": 1112539136 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033481444332999, + "loss": 3.0138, + "theoretical_loss": 3.6126813109054643, + "tokens_seen": 1112604672 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033480441323971915, + "loss": 2.7494, + "theoretical_loss": 3.6126614577306047, + "tokens_seen": 1112670208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347943831494484, + "loss": 2.7871, + "theoretical_loss": 3.6126416060524535, + "tokens_seen": 1112735744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347843530591775, + "loss": 2.7748, + "theoretical_loss": 3.6126217558708085, + "tokens_seen": 1112801280 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033477432296890675, + "loss": 2.8669, + "theoretical_loss": 3.6126019071854696, + "tokens_seen": 1112866816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033476429287863593, + "loss": 2.4874, + "theoretical_loss": 3.612582059996236, + "tokens_seen": 1112932352 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347542627883651, + "loss": 2.7502, + "theoretical_loss": 3.6125622143029066, + "tokens_seen": 1112997888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347442326980943, + "loss": 2.6639, + "theoretical_loss": 3.6125423701052806, + "tokens_seen": 1113063424 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033473420260782353, + "loss": 2.7515, + "theoretical_loss": 3.612522527403157, + "tokens_seen": 1113128960 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033472417251755265, + "loss": 2.6853, + "theoretical_loss": 3.6125026861963354, + "tokens_seen": 1113194496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003347141424272819, + "loss": 2.7526, + "theoretical_loss": 3.612482846484615, + "tokens_seen": 1113260032 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334704112337011, + "loss": 2.6622, + "theoretical_loss": 3.6124630082677944, + "tokens_seen": 1113325568 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033469408224674025, + "loss": 2.6131, + "theoretical_loss": 3.6124431715456744, + "tokens_seen": 1113391104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033468405215646943, + "loss": 2.6075, + "theoretical_loss": 3.612423336318053, + "tokens_seen": 1113456640 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346740220661986, + "loss": 2.7197, + "theoretical_loss": 3.6124035025847308, + "tokens_seen": 1113522176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346639919759278, + "loss": 2.5603, + "theoretical_loss": 3.6123836703455066, + "tokens_seen": 1113587712 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334653961885657, + "loss": 3.03, + "theoretical_loss": 3.6123638396001807, + "tokens_seen": 1113653248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033464393179538616, + "loss": 2.8079, + "theoretical_loss": 3.6123440103485516, + "tokens_seen": 1113718784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346339017051154, + "loss": 2.8281, + "theoretical_loss": 3.6123241825904193, + "tokens_seen": 1113784320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346238716148445, + "loss": 2.4955, + "theoretical_loss": 3.6123043563255837, + "tokens_seen": 1113849856 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033461384152457376, + "loss": 2.8836, + "theoretical_loss": 3.6122845315538443, + "tokens_seen": 1113915392 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003346038114343029, + "loss": 2.9342, + "theoretical_loss": 3.6122647082750006, + "tokens_seen": 1113980928 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345937813440321, + "loss": 2.7755, + "theoretical_loss": 3.6122448864888526, + "tokens_seen": 1114046464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 618528, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8394312858581543, + "objective/train/theoretical_loss": 3.612225066195201, + "objective/train/tokens_used": 1134572000, + "theoretical_loss": 3.612225066195201, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345837512537613, + "loss": 2.8486, + "theoretical_loss": 3.612225066195201, + "tokens_seen": 1114112000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345737211634905, + "loss": 2.487, + "theoretical_loss": 3.6122052473938435, + "tokens_seen": 1114177536 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033456369107321966, + "loss": 2.8196, + "theoretical_loss": 3.612185430084582, + "tokens_seen": 1114243072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345536609829489, + "loss": 2.7459, + "theoretical_loss": 3.6121656142672154, + "tokens_seen": 1114308608 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334543630892678, + "loss": 2.6842, + "theoretical_loss": 3.612145799941544, + "tokens_seen": 1114374144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033453360080240726, + "loss": 2.6307, + "theoretical_loss": 3.612125987107367, + "tokens_seen": 1114439680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345235707121364, + "loss": 2.8632, + "theoretical_loss": 3.6121061757644863, + "tokens_seen": 1114505216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345135406218656, + "loss": 2.64, + "theoretical_loss": 3.6120863659127, + "tokens_seen": 1114570752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003345035105315948, + "loss": 2.6673, + "theoretical_loss": 3.612066557551809, + "tokens_seen": 1114636288 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334493480441324, + "loss": 2.9368, + "theoretical_loss": 3.6120467506816136, + "tokens_seen": 1114701824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033448345035105316, + "loss": 2.9501, + "theoretical_loss": 3.612026945301914, + "tokens_seen": 1114767360 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033447342026078235, + "loss": 2.9252, + "theoretical_loss": 3.6120071414125103, + "tokens_seen": 1114832896 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344633901705115, + "loss": 2.6258, + "theoretical_loss": 3.611987339013202, + "tokens_seen": 1114898432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033445336008024076, + "loss": 2.807, + "theoretical_loss": 3.611967538103791, + "tokens_seen": 1114963968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344433299899699, + "loss": 2.8856, + "theoretical_loss": 3.611947738684076, + "tokens_seen": 1115029504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344332998996991, + "loss": 2.7013, + "theoretical_loss": 3.6119279407538585, + "tokens_seen": 1115095040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033442326980942825, + "loss": 2.6865, + "theoretical_loss": 3.611908144312938, + "tokens_seen": 1115160576 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003344132397191575, + "loss": 2.9106, + "theoretical_loss": 3.6118883493611165, + "tokens_seen": 1115226112 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033440320962888667, + "loss": 2.7511, + "theoretical_loss": 3.6118685558981927, + "tokens_seen": 1115291648 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033439317953861585, + "loss": 2.6814, + "theoretical_loss": 3.611848763923968, + "tokens_seen": 1115357184 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033438314944834503, + "loss": 2.902, + "theoretical_loss": 3.611828973438243, + "tokens_seen": 1115422720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033437311935807427, + "loss": 2.7772, + "theoretical_loss": 3.611809184440818, + "tokens_seen": 1115488256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003343630892678034, + "loss": 2.8768, + "theoretical_loss": 3.6117893969314947, + "tokens_seen": 1115553792 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033435305917753263, + "loss": 2.3855, + "theoretical_loss": 3.6117696109100716, + "tokens_seen": 1115619328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033434302908726175, + "loss": 2.8104, + "theoretical_loss": 3.611749826376352, + "tokens_seen": 1115684864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 619820, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4851090908050537, + "objective/train/theoretical_loss": 3.6117300433301347, + "objective/train/tokens_used": 1136210400, + "theoretical_loss": 3.6117300433301347, + "tokens_seen": 1115750400 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334332998996991, + "loss": 2.7111, + "theoretical_loss": 3.6117300433301347, + "tokens_seen": 1115750400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033432296890672017, + "loss": 2.8355, + "theoretical_loss": 3.611710261771221, + "tokens_seen": 1115815936 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033431293881644935, + "loss": 2.7384, + "theoretical_loss": 3.611690481699412, + "tokens_seen": 1115881472 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033430290872617853, + "loss": 2.6487, + "theoretical_loss": 3.611670703114509, + "tokens_seen": 1115947008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342928786359077, + "loss": 2.8993, + "theoretical_loss": 3.6116509260163125, + "tokens_seen": 1116012544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342828485456369, + "loss": 2.6758, + "theoretical_loss": 3.611631150404623, + "tokens_seen": 1116078080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033427281845536613, + "loss": 2.5941, + "theoretical_loss": 3.611611376279242, + "tokens_seen": 1116143616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033426278836509526, + "loss": 2.7803, + "theoretical_loss": 3.6115916036399702, + "tokens_seen": 1116209152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342527582748245, + "loss": 2.5758, + "theoretical_loss": 3.611571832486609, + "tokens_seen": 1116274688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342427281845536, + "loss": 2.6273, + "theoretical_loss": 3.6115520628189595, + "tokens_seen": 1116340224 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033423269809428285, + "loss": 2.9422, + "theoretical_loss": 3.611532294636823, + "tokens_seen": 1116405760 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033422266800401204, + "loss": 2.5773, + "theoretical_loss": 3.6115125279399996, + "tokens_seen": 1116471296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342126379137412, + "loss": 2.7014, + "theoretical_loss": 3.6114927627282922, + "tokens_seen": 1116536832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003342026078234704, + "loss": 2.6084, + "theoretical_loss": 3.611472999001501, + "tokens_seen": 1116602368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033419257773319963, + "loss": 2.8009, + "theoretical_loss": 3.611453236759427, + "tokens_seen": 1116667904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033418254764292876, + "loss": 2.757, + "theoretical_loss": 3.6114334760018725, + "tokens_seen": 1116733440 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334172517552658, + "loss": 2.8419, + "theoretical_loss": 3.6114137167286384, + "tokens_seen": 1116798976 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341624874623871, + "loss": 2.444, + "theoretical_loss": 3.6113939589395256, + "tokens_seen": 1116864512 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033415245737211636, + "loss": 2.7549, + "theoretical_loss": 3.6113742026343365, + "tokens_seen": 1116930048 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033414242728184554, + "loss": 2.7793, + "theoretical_loss": 3.6113544478128725, + "tokens_seen": 1116995584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341323971915747, + "loss": 2.4836, + "theoretical_loss": 3.6113346944749343, + "tokens_seen": 1117061120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341223671013039, + "loss": 2.7046, + "theoretical_loss": 3.611314942620324, + "tokens_seen": 1117126656 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003341123370110331, + "loss": 2.5733, + "theoretical_loss": 3.611295192248843, + "tokens_seen": 1117192192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033410230692076226, + "loss": 2.7553, + "theoretical_loss": 3.6112754433602934, + "tokens_seen": 1117257728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340922768304915, + "loss": 2.6006, + "theoretical_loss": 3.6112556959544766, + "tokens_seen": 1117323264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 620252, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6974408626556396, + "objective/train/theoretical_loss": 3.611235950031194, + "objective/train/tokens_used": 1137848800, + "theoretical_loss": 3.611235950031194, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340822467402207, + "loss": 2.6483, + "theoretical_loss": 3.611235950031194, + "tokens_seen": 1117388800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033407221664994986, + "loss": 2.8296, + "theoretical_loss": 3.6112162055902477, + "tokens_seen": 1117454336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340621865596791, + "loss": 2.6342, + "theoretical_loss": 3.61119646263144, + "tokens_seen": 1117519872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340521564694082, + "loss": 2.7249, + "theoretical_loss": 3.6111767211545716, + "tokens_seen": 1117585408 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033404212637913746, + "loss": 2.6615, + "theoretical_loss": 3.6111569811594455, + "tokens_seen": 1117650944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340320962888666, + "loss": 2.8549, + "theoretical_loss": 3.6111372426458628, + "tokens_seen": 1117716480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340220661985958, + "loss": 2.7704, + "theoretical_loss": 3.611117505613625, + "tokens_seen": 1117782016 + }, + { + "epoch": 0.34, + "learning_rate": 0.000334012036108325, + "loss": 2.79, + "theoretical_loss": 3.6110977700625355, + "tokens_seen": 1117847552 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003340020060180542, + "loss": 2.5227, + "theoretical_loss": 3.6110780359923957, + "tokens_seen": 1117913088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033399197592778336, + "loss": 3.0267, + "theoretical_loss": 3.6110583034030075, + "tokens_seen": 1117978624 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033398194583751255, + "loss": 2.4724, + "theoretical_loss": 3.6110385722941727, + "tokens_seen": 1118044160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339719157472417, + "loss": 2.6887, + "theoretical_loss": 3.611018842665694, + "tokens_seen": 1118109696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033396188565697096, + "loss": 2.7518, + "theoretical_loss": 3.6109991145173734, + "tokens_seen": 1118175232 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339518555667001, + "loss": 2.5372, + "theoretical_loss": 3.6109793878490133, + "tokens_seen": 1118240768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339418254764293, + "loss": 2.7651, + "theoretical_loss": 3.6109596626604157, + "tokens_seen": 1118306304 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033393179538615845, + "loss": 2.7977, + "theoretical_loss": 3.6109399389513825, + "tokens_seen": 1118371840 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003339217652958877, + "loss": 2.728, + "theoretical_loss": 3.610920216721717, + "tokens_seen": 1118437376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033391173520561687, + "loss": 2.6523, + "theoretical_loss": 3.610900495971221, + "tokens_seen": 1118502912 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033390170511534605, + "loss": 2.7489, + "theoretical_loss": 3.6108807766996964, + "tokens_seen": 1118568448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033389167502507523, + "loss": 2.8485, + "theoretical_loss": 3.6108610589069468, + "tokens_seen": 1118633984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033388164493480447, + "loss": 2.7418, + "theoretical_loss": 3.610841342592774, + "tokens_seen": 1118699520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338716148445336, + "loss": 2.6925, + "theoretical_loss": 3.6108216277569802, + "tokens_seen": 1118765056 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033386158475426283, + "loss": 2.8039, + "theoretical_loss": 3.610801914399369, + "tokens_seen": 1118830592 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033385155466399195, + "loss": 2.7368, + "theoretical_loss": 3.610782202519742, + "tokens_seen": 1118896128 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338415245737212, + "loss": 2.8413, + "theoretical_loss": 3.6107624921179013, + "tokens_seen": 1118961664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 621194, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.00057315826416, + "objective/train/theoretical_loss": 3.6107427831936514, + "objective/train/tokens_used": 1139487200, + "theoretical_loss": 3.6107427831936514, + "tokens_seen": 1119027200 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033383149448345037, + "loss": 2.9902, + "theoretical_loss": 3.6107427831936514, + "tokens_seen": 1119027200 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033382146439317955, + "loss": 2.743, + "theoretical_loss": 3.610723075746794, + "tokens_seen": 1119092736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033381143430290873, + "loss": 2.6116, + "theoretical_loss": 3.6107033697771316, + "tokens_seen": 1119158272 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003338014042126379, + "loss": 3.0118, + "theoretical_loss": 3.6106836652844674, + "tokens_seen": 1119223808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337913741223671, + "loss": 2.5351, + "theoretical_loss": 3.610663962268604, + "tokens_seen": 1119289344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033378134403209633, + "loss": 2.6608, + "theoretical_loss": 3.6106442607293445, + "tokens_seen": 1119354880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033377131394182546, + "loss": 2.6882, + "theoretical_loss": 3.6106245606664915, + "tokens_seen": 1119420416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337612838515547, + "loss": 2.6599, + "theoretical_loss": 3.610604862079848, + "tokens_seen": 1119485952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337512537612838, + "loss": 2.7627, + "theoretical_loss": 3.6105851649692173, + "tokens_seen": 1119551488 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033374122367101306, + "loss": 2.6458, + "theoretical_loss": 3.610565469334402, + "tokens_seen": 1119617024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033373119358074224, + "loss": 2.6959, + "theoretical_loss": 3.610545775175205, + "tokens_seen": 1119682560 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337211634904714, + "loss": 2.746, + "theoretical_loss": 3.61052608249143, + "tokens_seen": 1119748096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003337111334002006, + "loss": 2.6225, + "theoretical_loss": 3.61050639128288, + "tokens_seen": 1119813632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033370110330992983, + "loss": 2.785, + "theoretical_loss": 3.610486701549357, + "tokens_seen": 1119879168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033369107321965896, + "loss": 2.8137, + "theoretical_loss": 3.610467013290666, + "tokens_seen": 1119944704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336810431293882, + "loss": 2.9026, + "theoretical_loss": 3.61044732650661, + "tokens_seen": 1120010240 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336710130391173, + "loss": 2.7828, + "theoretical_loss": 3.6104276411969907, + "tokens_seen": 1120075776 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033366098294884656, + "loss": 2.7948, + "theoretical_loss": 3.6104079573616126, + "tokens_seen": 1120141312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033365095285857574, + "loss": 2.748, + "theoretical_loss": 3.610388275000279, + "tokens_seen": 1120206848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336409227683049, + "loss": 2.8541, + "theoretical_loss": 3.610368594112793, + "tokens_seen": 1120272384 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336308926780341, + "loss": 2.7131, + "theoretical_loss": 3.6103489146989585, + "tokens_seen": 1120337920 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336208625877633, + "loss": 2.4622, + "theoretical_loss": 3.610329236758578, + "tokens_seen": 1120403456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033361083249749246, + "loss": 2.7207, + "theoretical_loss": 3.6103095602914563, + "tokens_seen": 1120468992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003336008024072217, + "loss": 2.6971, + "theoretical_loss": 3.610289885297396, + "tokens_seen": 1120534528 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335907723169508, + "loss": 3.0744, + "theoretical_loss": 3.610270211776201, + "tokens_seen": 1120600064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 621844, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9886622428894043, + "objective/train/theoretical_loss": 3.6102505397276743, + "objective/train/tokens_used": 1141125600, + "theoretical_loss": 3.6102505397276743, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033358074222668006, + "loss": 2.812, + "theoretical_loss": 3.6102505397276743, + "tokens_seen": 1120665600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033357071213640924, + "loss": 2.6918, + "theoretical_loss": 3.6102308691516205, + "tokens_seen": 1120731136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335606820461384, + "loss": 2.6555, + "theoretical_loss": 3.610211200047843, + "tokens_seen": 1120796672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335506519558676, + "loss": 2.9233, + "theoretical_loss": 3.6101915324161453, + "tokens_seen": 1120862208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335406218655968, + "loss": 2.6204, + "theoretical_loss": 3.6101718662563314, + "tokens_seen": 1120927744 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033353059177532597, + "loss": 2.5356, + "theoretical_loss": 3.610152201568205, + "tokens_seen": 1120993280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003335205616850552, + "loss": 2.8194, + "theoretical_loss": 3.6101325383515697, + "tokens_seen": 1121058816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033351053159478433, + "loss": 2.8566, + "theoretical_loss": 3.6101128766062294, + "tokens_seen": 1121124352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033350050150451356, + "loss": 2.7643, + "theoretical_loss": 3.6100932163319888, + "tokens_seen": 1121189888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334904714142427, + "loss": 2.7626, + "theoretical_loss": 3.6100735575286507, + "tokens_seen": 1121255424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334804413239719, + "loss": 2.5128, + "theoretical_loss": 3.6100539001960197, + "tokens_seen": 1121320960 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334704112337011, + "loss": 2.7467, + "theoretical_loss": 3.6100342443339004, + "tokens_seen": 1121386496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334603811434303, + "loss": 2.7479, + "theoretical_loss": 3.6100145899420957, + "tokens_seen": 1121452032 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033345035105315947, + "loss": 2.8238, + "theoretical_loss": 3.6099949370204105, + "tokens_seen": 1121517568 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033344032096288865, + "loss": 2.6709, + "theoretical_loss": 3.6099752855686487, + "tokens_seen": 1121583104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033343029087261783, + "loss": 2.7122, + "theoretical_loss": 3.6099556355866143, + "tokens_seen": 1121648640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033342026078234707, + "loss": 2.7043, + "theoretical_loss": 3.6099359870741115, + "tokens_seen": 1121714176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003334102306920762, + "loss": 2.6631, + "theoretical_loss": 3.609916340030945, + "tokens_seen": 1121779712 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033340020060180543, + "loss": 2.5839, + "theoretical_loss": 3.609896694456919, + "tokens_seen": 1121845248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333901705115346, + "loss": 2.9894, + "theoretical_loss": 3.6098770503518374, + "tokens_seen": 1121910784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333801404212638, + "loss": 2.6817, + "theoretical_loss": 3.609857407715505, + "tokens_seen": 1121976320 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333370110330993, + "loss": 2.2964, + "theoretical_loss": 3.609837766547726, + "tokens_seen": 1122041856 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033336008024072215, + "loss": 2.6476, + "theoretical_loss": 3.609818126848305, + "tokens_seen": 1122107392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033335005015045134, + "loss": 2.7987, + "theoretical_loss": 3.609798488617046, + "tokens_seen": 1122172928 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033334002006018057, + "loss": 2.6787, + "theoretical_loss": 3.609778851853754, + "tokens_seen": 1122238464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 622614, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.409770965576172, + "objective/train/theoretical_loss": 3.609759216558233, + "objective/train/tokens_used": 1142764000, + "theoretical_loss": 3.609759216558233, + "tokens_seen": 1122304000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033332998996990975, + "loss": 2.6987, + "theoretical_loss": 3.609759216558233, + "tokens_seen": 1122304000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033331995987963893, + "loss": 2.4774, + "theoretical_loss": 3.6097395827302887, + "tokens_seen": 1122369536 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003333099297893681, + "loss": 2.7174, + "theoretical_loss": 3.609719950369725, + "tokens_seen": 1122435072 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332998996990973, + "loss": 2.8698, + "theoretical_loss": 3.609700319476346, + "tokens_seen": 1122500608 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033328986960882653, + "loss": 2.7169, + "theoretical_loss": 3.6096806900499576, + "tokens_seen": 1122566144 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033327983951855566, + "loss": 2.6388, + "theoretical_loss": 3.6096610620903635, + "tokens_seen": 1122631680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332698094282849, + "loss": 2.608, + "theoretical_loss": 3.609641435597369, + "tokens_seen": 1122697216 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333259779338014, + "loss": 2.9848, + "theoretical_loss": 3.609621810570779, + "tokens_seen": 1122762752 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033324974924774326, + "loss": 2.5629, + "theoretical_loss": 3.609602187010398, + "tokens_seen": 1122828288 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033323971915747244, + "loss": 2.6, + "theoretical_loss": 3.6095825649160314, + "tokens_seen": 1122893824 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332296890672016, + "loss": 2.7688, + "theoretical_loss": 3.6095629442874833, + "tokens_seen": 1122959360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003332196589769308, + "loss": 2.555, + "theoretical_loss": 3.6095433251245597, + "tokens_seen": 1123024896 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033320962888666003, + "loss": 2.859, + "theoretical_loss": 3.6095237074270647, + "tokens_seen": 1123090432 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033319959879638916, + "loss": 2.8304, + "theoretical_loss": 3.609504091194804, + "tokens_seen": 1123155968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331895687061184, + "loss": 2.5043, + "theoretical_loss": 3.609484476427582, + "tokens_seen": 1123221504 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331795386158475, + "loss": 2.7801, + "theoretical_loss": 3.609464863125204, + "tokens_seen": 1123287040 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033316950852557676, + "loss": 2.6406, + "theoretical_loss": 3.609445251287476, + "tokens_seen": 1123352576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033315947843530594, + "loss": 2.6735, + "theoretical_loss": 3.609425640914202, + "tokens_seen": 1123418112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331494483450351, + "loss": 2.6454, + "theoretical_loss": 3.609406032005188, + "tokens_seen": 1123483648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331394182547643, + "loss": 2.6867, + "theoretical_loss": 3.609386424560239, + "tokens_seen": 1123549184 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331293881644935, + "loss": 2.5992, + "theoretical_loss": 3.6093668185791605, + "tokens_seen": 1123614720 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033311935807422266, + "loss": 2.6658, + "theoretical_loss": 3.6093472140617573, + "tokens_seen": 1123680256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003331093279839519, + "loss": 2.7022, + "theoretical_loss": 3.609327611007835, + "tokens_seen": 1123745792 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333099297893681, + "loss": 2.7329, + "theoretical_loss": 3.6093080094172, + "tokens_seen": 1123811328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033308926780341026, + "loss": 2.5589, + "theoretical_loss": 3.6092884092896558, + "tokens_seen": 1123876864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 624048, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.0764658451080322, + "objective/train/theoretical_loss": 3.60926881062501, + "objective/train/tokens_used": 1144402400, + "theoretical_loss": 3.60926881062501, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033307923771313944, + "loss": 2.8697, + "theoretical_loss": 3.60926881062501, + "tokens_seen": 1123942400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330692076228686, + "loss": 2.7104, + "theoretical_loss": 3.609249213423066, + "tokens_seen": 1124007936 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330591775325978, + "loss": 2.8209, + "theoretical_loss": 3.6092296176836314, + "tokens_seen": 1124073472 + }, + { + "epoch": 0.34, + "learning_rate": 0.000333049147442327, + "loss": 2.7686, + "theoretical_loss": 3.6092100234065105, + "tokens_seen": 1124139008 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033303911735205617, + "loss": 2.6625, + "theoretical_loss": 3.609190430591509, + "tokens_seen": 1124204544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003330290872617854, + "loss": 2.7168, + "theoretical_loss": 3.609170839238433, + "tokens_seen": 1124270080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033301905717151453, + "loss": 2.9399, + "theoretical_loss": 3.6091512493470885, + "tokens_seen": 1124335616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033300902708124376, + "loss": 2.7011, + "theoretical_loss": 3.60913166091728, + "tokens_seen": 1124401152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329989969909729, + "loss": 2.6836, + "theoretical_loss": 3.6091120739488147, + "tokens_seen": 1124466688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329889669007021, + "loss": 2.8206, + "theoretical_loss": 3.609092488441498, + "tokens_seen": 1124532224 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329789368104313, + "loss": 2.6126, + "theoretical_loss": 3.609072904395135, + "tokens_seen": 1124597760 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329689067201605, + "loss": 2.6087, + "theoretical_loss": 3.609053321809532, + "tokens_seen": 1124663296 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033295887662988967, + "loss": 2.7225, + "theoretical_loss": 3.609033740684496, + "tokens_seen": 1124728832 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033294884653961885, + "loss": 2.9933, + "theoretical_loss": 3.6090141610198314, + "tokens_seen": 1124794368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033293881644934803, + "loss": 2.831, + "theoretical_loss": 3.6089945828153454, + "tokens_seen": 1124859904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033292878635907727, + "loss": 2.7406, + "theoretical_loss": 3.608975006070843, + "tokens_seen": 1124925440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003329187562688064, + "loss": 2.6774, + "theoretical_loss": 3.608955430786131, + "tokens_seen": 1124990976 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033290872617853563, + "loss": 2.7052, + "theoretical_loss": 3.6089358569610153, + "tokens_seen": 1125056512 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328986960882648, + "loss": 2.6838, + "theoretical_loss": 3.608916284595302, + "tokens_seen": 1125122048 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332888665997994, + "loss": 2.7067, + "theoretical_loss": 3.6088967136887975, + "tokens_seen": 1125187584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328786359077232, + "loss": 2.7984, + "theoretical_loss": 3.608877144241308, + "tokens_seen": 1125253120 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033286860581745235, + "loss": 2.9188, + "theoretical_loss": 3.6088575762526394, + "tokens_seen": 1125318656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033285857572718154, + "loss": 2.7095, + "theoretical_loss": 3.6088380097225983, + "tokens_seen": 1125384192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033284854563691077, + "loss": 2.734, + "theoretical_loss": 3.608818444650991, + "tokens_seen": 1125449728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328385155466399, + "loss": 2.5812, + "theoretical_loss": 3.6087988810376235, + "tokens_seen": 1125515264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 624451, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6627368927001953, + "objective/train/theoretical_loss": 3.608779318882303, + "objective/train/tokens_used": 1146040800, + "theoretical_loss": 3.608779318882303, + "tokens_seen": 1125580800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033282848545636913, + "loss": 2.6308, + "theoretical_loss": 3.608779318882303, + "tokens_seen": 1125580800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033281845536609826, + "loss": 2.5499, + "theoretical_loss": 3.608759758184836, + "tokens_seen": 1125646336 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003328084252758275, + "loss": 2.8517, + "theoretical_loss": 3.6087401989450276, + "tokens_seen": 1125711872 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327983951855567, + "loss": 2.6624, + "theoretical_loss": 3.6087206411626855, + "tokens_seen": 1125777408 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033278836509528586, + "loss": 2.751, + "theoretical_loss": 3.6087010848376164, + "tokens_seen": 1125842944 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033277833500501504, + "loss": 2.5501, + "theoretical_loss": 3.608681529969626, + "tokens_seen": 1125908480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327683049147442, + "loss": 2.6393, + "theoretical_loss": 3.6086619765585217, + "tokens_seen": 1125974016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327582748244734, + "loss": 2.6149, + "theoretical_loss": 3.6086424246041098, + "tokens_seen": 1126039552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033274824473420264, + "loss": 2.805, + "theoretical_loss": 3.6086228741061976, + "tokens_seen": 1126105088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033273821464393176, + "loss": 2.7527, + "theoretical_loss": 3.608603325064591, + "tokens_seen": 1126170624 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332728184553661, + "loss": 2.9454, + "theoretical_loss": 3.608583777479097, + "tokens_seen": 1126236160 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003327181544633902, + "loss": 2.8227, + "theoretical_loss": 3.6085642313495225, + "tokens_seen": 1126301696 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033270812437311936, + "loss": 2.7314, + "theoretical_loss": 3.6085446866756747, + "tokens_seen": 1126367232 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033269809428284854, + "loss": 2.6964, + "theoretical_loss": 3.60852514345736, + "tokens_seen": 1126432768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326880641925777, + "loss": 2.8856, + "theoretical_loss": 3.6085056016943855, + "tokens_seen": 1126498304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326780341023069, + "loss": 2.9844, + "theoretical_loss": 3.608486061386558, + "tokens_seen": 1126563840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033266800401203614, + "loss": 2.8392, + "theoretical_loss": 3.6084665225336856, + "tokens_seen": 1126629376 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033265797392176527, + "loss": 2.8072, + "theoretical_loss": 3.608446985135574, + "tokens_seen": 1126694912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326479438314945, + "loss": 2.9037, + "theoretical_loss": 3.6084274491920305, + "tokens_seen": 1126760448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033263791374122363, + "loss": 2.713, + "theoretical_loss": 3.6084079147028625, + "tokens_seen": 1126825984 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033262788365095286, + "loss": 2.622, + "theoretical_loss": 3.608388381667877, + "tokens_seen": 1126891520 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033261785356068205, + "loss": 2.7095, + "theoretical_loss": 3.6083688500868814, + "tokens_seen": 1126957056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003326078234704112, + "loss": 2.6724, + "theoretical_loss": 3.6083493199596828, + "tokens_seen": 1127022592 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325977933801404, + "loss": 2.6232, + "theoretical_loss": 3.6083297912860886, + "tokens_seen": 1127088128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033258776328986964, + "loss": 2.7234, + "theoretical_loss": 3.6083102640659055, + "tokens_seen": 1127153664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 625621, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.037780284881592, + "objective/train/theoretical_loss": 3.608290738298942, + "objective/train/tokens_used": 1147679200, + "theoretical_loss": 3.608290738298942, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325777331995988, + "loss": 2.6553, + "theoretical_loss": 3.608290738298942, + "tokens_seen": 1127219200 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332567703109328, + "loss": 2.813, + "theoretical_loss": 3.608271213985004, + "tokens_seen": 1127284736 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325576730190572, + "loss": 2.5845, + "theoretical_loss": 3.6082516911239004, + "tokens_seen": 1127350272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033254764292878637, + "loss": 2.8016, + "theoretical_loss": 3.608232169715438, + "tokens_seen": 1127415808 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325376128385156, + "loss": 2.8117, + "theoretical_loss": 3.6082126497594236, + "tokens_seen": 1127481344 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033252758274824473, + "loss": 2.7862, + "theoretical_loss": 3.6081931312556654, + "tokens_seen": 1127546880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033251755265797396, + "loss": 2.8977, + "theoretical_loss": 3.608173614203971, + "tokens_seen": 1127612416 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003325075225677031, + "loss": 2.6172, + "theoretical_loss": 3.608154098604148, + "tokens_seen": 1127677952 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033249749247743233, + "loss": 2.8162, + "theoretical_loss": 3.6081345844560033, + "tokens_seen": 1127743488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324874623871615, + "loss": 2.5889, + "theoretical_loss": 3.6081150717593458, + "tokens_seen": 1127809024 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324774322968907, + "loss": 2.8404, + "theoretical_loss": 3.6080955605139824, + "tokens_seen": 1127874560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033246740220661987, + "loss": 2.7708, + "theoretical_loss": 3.6080760507197214, + "tokens_seen": 1127940096 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033245737211634905, + "loss": 2.7785, + "theoretical_loss": 3.6080565423763695, + "tokens_seen": 1128005632 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033244734202607823, + "loss": 2.9172, + "theoretical_loss": 3.6080370354837354, + "tokens_seen": 1128071168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033243731193580747, + "loss": 2.5575, + "theoretical_loss": 3.608017530041627, + "tokens_seen": 1128136704 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003324272818455366, + "loss": 2.6802, + "theoretical_loss": 3.6079980260498514, + "tokens_seen": 1128202240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033241725175526583, + "loss": 2.6072, + "theoretical_loss": 3.6079785235082174, + "tokens_seen": 1128267776 + }, + { + "epoch": 0.34, + "learning_rate": 0.000332407221664995, + "loss": 2.6779, + "theoretical_loss": 3.6079590224165328, + "tokens_seen": 1128333312 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323971915747242, + "loss": 2.6611, + "theoretical_loss": 3.607939522774605, + "tokens_seen": 1128398848 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323871614844534, + "loss": 2.7856, + "theoretical_loss": 3.607920024582242, + "tokens_seen": 1128464384 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033237713139418255, + "loss": 2.6113, + "theoretical_loss": 3.607900527839253, + "tokens_seen": 1128529920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033236710130391174, + "loss": 2.8294, + "theoretical_loss": 3.6078810325454453, + "tokens_seen": 1128595456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033235707121364097, + "loss": 2.7486, + "theoretical_loss": 3.607861538700627, + "tokens_seen": 1128660992 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323470411233701, + "loss": 2.9031, + "theoretical_loss": 3.607842046304606, + "tokens_seen": 1128726528 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033233701103309933, + "loss": 2.8622, + "theoretical_loss": 3.607822555357192, + "tokens_seen": 1128792064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 626861, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9778614044189453, + "objective/train/theoretical_loss": 3.6078030658581914, + "objective/train/tokens_used": 1149317600, + "theoretical_loss": 3.6078030658581914, + "tokens_seen": 1128857600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033232698094282846, + "loss": 2.7505, + "theoretical_loss": 3.6078030658581914, + "tokens_seen": 1128857600 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323169508525577, + "loss": 2.6772, + "theoretical_loss": 3.607783577807413, + "tokens_seen": 1128923136 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003323069207622869, + "loss": 2.654, + "theoretical_loss": 3.6077640912046665, + "tokens_seen": 1128988672 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033229689067201606, + "loss": 2.6944, + "theoretical_loss": 3.607744606049758, + "tokens_seen": 1129054208 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033228686058174524, + "loss": 2.7522, + "theoretical_loss": 3.6077251223424973, + "tokens_seen": 1129119744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322768304914744, + "loss": 2.7485, + "theoretical_loss": 3.607705640082693, + "tokens_seen": 1129185280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322668004012036, + "loss": 2.78, + "theoretical_loss": 3.607686159270153, + "tokens_seen": 1129250816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033225677031093284, + "loss": 2.8463, + "theoretical_loss": 3.6076666799046855, + "tokens_seen": 1129316352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033224674022066196, + "loss": 2.7895, + "theoretical_loss": 3.6076472019861, + "tokens_seen": 1129381888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322367101303912, + "loss": 2.5306, + "theoretical_loss": 3.607627725514204, + "tokens_seen": 1129447424 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003322266800401204, + "loss": 2.5914, + "theoretical_loss": 3.6076082504888074, + "tokens_seen": 1129512960 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033221664994984956, + "loss": 2.6608, + "theoretical_loss": 3.6075887769097177, + "tokens_seen": 1129578496 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033220661985957874, + "loss": 2.8178, + "theoretical_loss": 3.607569304776744, + "tokens_seen": 1129644032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321965897693079, + "loss": 2.6823, + "theoretical_loss": 3.607549834089695, + "tokens_seen": 1129709568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321865596790371, + "loss": 2.7135, + "theoretical_loss": 3.6075303648483796, + "tokens_seen": 1129775104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033217652958876634, + "loss": 2.7891, + "theoretical_loss": 3.607510897052607, + "tokens_seen": 1129840640 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033216649949849547, + "loss": 2.5299, + "theoretical_loss": 3.6074914307021846, + "tokens_seen": 1129906176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321564694082247, + "loss": 2.7886, + "theoretical_loss": 3.6074719657969228, + "tokens_seen": 1129971712 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033214643931795383, + "loss": 2.8238, + "theoretical_loss": 3.60745250233663, + "tokens_seen": 1130037248 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033213640922768306, + "loss": 2.7838, + "theoretical_loss": 3.607433040321115, + "tokens_seen": 1130102784 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033212637913741225, + "loss": 2.7236, + "theoretical_loss": 3.6074135797501867, + "tokens_seen": 1130168320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321163490471414, + "loss": 2.9937, + "theoretical_loss": 3.6073941206236544, + "tokens_seen": 1130233856 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003321063189568706, + "loss": 2.8337, + "theoretical_loss": 3.607374662941327, + "tokens_seen": 1130299392 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033209628886659984, + "loss": 2.7751, + "theoretical_loss": 3.607355206703013, + "tokens_seen": 1130364928 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033208625877632897, + "loss": 2.7562, + "theoretical_loss": 3.607335751908522, + "tokens_seen": 1130430464 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 627282, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7507452964782715, + "objective/train/theoretical_loss": 3.6073162985576643, + "objective/train/tokens_used": 1150956000, + "theoretical_loss": 3.6073162985576643, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320762286860582, + "loss": 2.7378, + "theoretical_loss": 3.6073162985576643, + "tokens_seen": 1130496000 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033206619859578733, + "loss": 2.6933, + "theoretical_loss": 3.6072968466502475, + "tokens_seen": 1130561536 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033205616850551657, + "loss": 2.6401, + "theoretical_loss": 3.6072773961860816, + "tokens_seen": 1130627072 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033204613841524575, + "loss": 2.7799, + "theoretical_loss": 3.6072579471649755, + "tokens_seen": 1130692608 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033203610832497493, + "loss": 2.6876, + "theoretical_loss": 3.607238499586739, + "tokens_seen": 1130758144 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320260782347041, + "loss": 2.7543, + "theoretical_loss": 3.6072190534511805, + "tokens_seen": 1130823680 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320160481444333, + "loss": 2.8909, + "theoretical_loss": 3.60719960875811, + "tokens_seen": 1130889216 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003320060180541625, + "loss": 2.6733, + "theoretical_loss": 3.6071801655073377, + "tokens_seen": 1130954752 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319959879638917, + "loss": 2.5583, + "theoretical_loss": 3.6071607236986716, + "tokens_seen": 1131020288 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033198595787362084, + "loss": 2.9844, + "theoretical_loss": 3.607141283331922, + "tokens_seen": 1131085824 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033197592778335007, + "loss": 2.933, + "theoretical_loss": 3.6071218444068984, + "tokens_seen": 1131151360 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319658976930792, + "loss": 2.4985, + "theoretical_loss": 3.60710240692341, + "tokens_seen": 1131216896 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033195586760280843, + "loss": 2.7601, + "theoretical_loss": 3.607082970881267, + "tokens_seen": 1131282432 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319458375125376, + "loss": 2.9582, + "theoretical_loss": 3.607063536280278, + "tokens_seen": 1131347968 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319358074222668, + "loss": 2.6208, + "theoretical_loss": 3.607044103120254, + "tokens_seen": 1131413504 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331925777331996, + "loss": 2.9075, + "theoretical_loss": 3.6070246714010037, + "tokens_seen": 1131479040 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003319157472417252, + "loss": 2.8108, + "theoretical_loss": 3.6070052411223372, + "tokens_seen": 1131544576 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033190571715145434, + "loss": 2.81, + "theoretical_loss": 3.6069858122840643, + "tokens_seen": 1131610112 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318956870611836, + "loss": 2.696, + "theoretical_loss": 3.606966384885995, + "tokens_seen": 1131675648 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318856569709127, + "loss": 2.6364, + "theoretical_loss": 3.6069469589279386, + "tokens_seen": 1131741184 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033187562688064194, + "loss": 2.7666, + "theoretical_loss": 3.6069275344097047, + "tokens_seen": 1131806720 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318655967903711, + "loss": 2.8148, + "theoretical_loss": 3.606908111331105, + "tokens_seen": 1131872256 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318555667001003, + "loss": 2.9294, + "theoretical_loss": 3.6068886896919476, + "tokens_seen": 1131937792 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318455366098295, + "loss": 2.6665, + "theoretical_loss": 3.6068692694920426, + "tokens_seen": 1132003328 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033183550651955866, + "loss": 2.5168, + "theoretical_loss": 3.6068498507312015, + "tokens_seen": 1132068864 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 628386, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4533891677856445, + "objective/train/theoretical_loss": 3.6068304334092334, + "objective/train/tokens_used": 1152594400, + "theoretical_loss": 3.6068304334092334, + "tokens_seen": 1132134400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318254764292879, + "loss": 2.6008, + "theoretical_loss": 3.6068304334092334, + "tokens_seen": 1132134400 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003318154463390171, + "loss": 2.8305, + "theoretical_loss": 3.606811017525948, + "tokens_seen": 1132199936 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033180541624874626, + "loss": 2.7182, + "theoretical_loss": 3.606791603081156, + "tokens_seen": 1132265472 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033179538615847544, + "loss": 2.8093, + "theoretical_loss": 3.6067721900746674, + "tokens_seen": 1132331008 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317853560682046, + "loss": 2.5651, + "theoretical_loss": 3.6067527785062925, + "tokens_seen": 1132396544 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317753259779338, + "loss": 2.8494, + "theoretical_loss": 3.606733368375842, + "tokens_seen": 1132462080 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033176529588766304, + "loss": 2.7811, + "theoretical_loss": 3.606713959683125, + "tokens_seen": 1132527616 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033175526579739216, + "loss": 2.9312, + "theoretical_loss": 3.6066945524279532, + "tokens_seen": 1132593152 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317452357071214, + "loss": 2.7798, + "theoretical_loss": 3.6066751466101357, + "tokens_seen": 1132658688 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317352056168506, + "loss": 2.7387, + "theoretical_loss": 3.606655742229483, + "tokens_seen": 1132724224 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033172517552657976, + "loss": 2.6532, + "theoretical_loss": 3.606636339285807, + "tokens_seen": 1132789760 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033171514543630894, + "loss": 2.7307, + "theoretical_loss": 3.6066169377789166, + "tokens_seen": 1132855296 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003317051153460381, + "loss": 2.7641, + "theoretical_loss": 3.6065975377086232, + "tokens_seen": 1132920832 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316950852557673, + "loss": 2.7869, + "theoretical_loss": 3.6065781390747365, + "tokens_seen": 1132986368 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033168505516549654, + "loss": 2.6226, + "theoretical_loss": 3.606558741877068, + "tokens_seen": 1133051904 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033167502507522567, + "loss": 2.7482, + "theoretical_loss": 3.6065393461154276, + "tokens_seen": 1133117440 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316649949849549, + "loss": 2.4595, + "theoretical_loss": 3.6065199517896263, + "tokens_seen": 1133182976 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033165496489468403, + "loss": 2.7327, + "theoretical_loss": 3.6065005588994747, + "tokens_seen": 1133248512 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033164493480441326, + "loss": 2.7135, + "theoretical_loss": 3.606481167444783, + "tokens_seen": 1133314048 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033163490471414245, + "loss": 2.6186, + "theoretical_loss": 3.6064617774253627, + "tokens_seen": 1133379584 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316248746238716, + "loss": 2.9134, + "theoretical_loss": 3.606442388841024, + "tokens_seen": 1133445120 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003316148445336008, + "loss": 2.8247, + "theoretical_loss": 3.606423001691579, + "tokens_seen": 1133510656 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033160481444333004, + "loss": 2.6597, + "theoretical_loss": 3.606403615976837, + "tokens_seen": 1133576192 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033159478435305917, + "loss": 2.4916, + "theoretical_loss": 3.606384231696609, + "tokens_seen": 1133641728 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315847542627884, + "loss": 2.9193, + "theoretical_loss": 3.6063648488507067, + "tokens_seen": 1133707264 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 629006, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7128870487213135, + "objective/train/theoretical_loss": 3.606345467438941, + "objective/train/tokens_used": 1154232800, + "theoretical_loss": 3.606345467438941, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033157472417251753, + "loss": 2.7728, + "theoretical_loss": 3.606345467438941, + "tokens_seen": 1133772800 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033156469408224677, + "loss": 2.7129, + "theoretical_loss": 3.6063260874611225, + "tokens_seen": 1133838336 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033155466399197595, + "loss": 2.7891, + "theoretical_loss": 3.6063067089170624, + "tokens_seen": 1133903872 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033154463390170513, + "loss": 2.6031, + "theoretical_loss": 3.6062873318065716, + "tokens_seen": 1133969408 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315346038114343, + "loss": 2.3994, + "theoretical_loss": 3.6062679561294617, + "tokens_seen": 1134034944 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315245737211635, + "loss": 2.7607, + "theoretical_loss": 3.606248581885543, + "tokens_seen": 1134100480 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315145436308927, + "loss": 2.7066, + "theoretical_loss": 3.6062292090746277, + "tokens_seen": 1134166016 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003315045135406219, + "loss": 2.6768, + "theoretical_loss": 3.606209837696527, + "tokens_seen": 1134231552 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033149448345035104, + "loss": 2.6928, + "theoretical_loss": 3.6061904677510506, + "tokens_seen": 1134297088 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033148445336008027, + "loss": 2.6556, + "theoretical_loss": 3.6061710992380114, + "tokens_seen": 1134362624 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314744232698094, + "loss": 2.9638, + "theoretical_loss": 3.60615173215722, + "tokens_seen": 1134428160 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033146439317953863, + "loss": 2.7812, + "theoretical_loss": 3.606132366508488, + "tokens_seen": 1134493696 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314543630892678, + "loss": 2.5463, + "theoretical_loss": 3.6061130022916266, + "tokens_seen": 1134559232 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331444332998997, + "loss": 2.8497, + "theoretical_loss": 3.6060936395064473, + "tokens_seen": 1134624768 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314343029087262, + "loss": 2.7433, + "theoretical_loss": 3.606074278152762, + "tokens_seen": 1134690304 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314242728184554, + "loss": 2.9172, + "theoretical_loss": 3.606054918230382, + "tokens_seen": 1134755840 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033141424272818454, + "loss": 2.6835, + "theoretical_loss": 3.6060355597391176, + "tokens_seen": 1134821376 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003314042126379138, + "loss": 2.6681, + "theoretical_loss": 3.6060162026787825, + "tokens_seen": 1134886912 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313941825476429, + "loss": 2.6483, + "theoretical_loss": 3.6059968470491865, + "tokens_seen": 1134952448 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033138415245737214, + "loss": 2.723, + "theoretical_loss": 3.6059774928501422, + "tokens_seen": 1135017984 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313741223671013, + "loss": 2.6665, + "theoretical_loss": 3.6059581400814613, + "tokens_seen": 1135083520 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313640922768305, + "loss": 2.8749, + "theoretical_loss": 3.6059387887429546, + "tokens_seen": 1135149056 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313540621865597, + "loss": 2.6138, + "theoretical_loss": 3.605919438834435, + "tokens_seen": 1135214592 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033134403209628886, + "loss": 2.7763, + "theoretical_loss": 3.605900090355714, + "tokens_seen": 1135280128 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033133400200601804, + "loss": 2.7442, + "theoretical_loss": 3.605880743306603, + "tokens_seen": 1135345664 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 630145, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.701599597930908, + "objective/train/theoretical_loss": 3.605861397686914, + "objective/train/tokens_used": 1155871200, + "theoretical_loss": 3.605861397686914, + "tokens_seen": 1135411200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313239719157473, + "loss": 2.7269, + "theoretical_loss": 3.605861397686914, + "tokens_seen": 1135411200 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003313139418254764, + "loss": 2.8759, + "theoretical_loss": 3.605842053496459, + "tokens_seen": 1135476736 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033130391173520564, + "loss": 2.8524, + "theoretical_loss": 3.60582271073505, + "tokens_seen": 1135542272 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033129388164493477, + "loss": 2.7067, + "theoretical_loss": 3.6058033694024987, + "tokens_seen": 1135607808 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331283851554664, + "loss": 2.7671, + "theoretical_loss": 3.605784029498617, + "tokens_seen": 1135673344 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312738214643932, + "loss": 2.5566, + "theoretical_loss": 3.605764691023218, + "tokens_seen": 1135738880 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033126379137412236, + "loss": 2.6674, + "theoretical_loss": 3.605745353976112, + "tokens_seen": 1135804416 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033125376128385155, + "loss": 3.0017, + "theoretical_loss": 3.605726018357113, + "tokens_seen": 1135869952 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312437311935808, + "loss": 2.8305, + "theoretical_loss": 3.6057066841660315, + "tokens_seen": 1135935488 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312337011033099, + "loss": 2.8144, + "theoretical_loss": 3.605687351402681, + "tokens_seen": 1136001024 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033122367101303914, + "loss": 2.9268, + "theoretical_loss": 3.6056680200668723, + "tokens_seen": 1136066560 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033121364092276827, + "loss": 2.7998, + "theoretical_loss": 3.605648690158419, + "tokens_seen": 1136132096 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003312036108324975, + "loss": 2.8524, + "theoretical_loss": 3.605629361677133, + "tokens_seen": 1136197632 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311935807422267, + "loss": 2.7364, + "theoretical_loss": 3.605610034622826, + "tokens_seen": 1136263168 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033118355065195587, + "loss": 2.8107, + "theoretical_loss": 3.6055907089953108, + "tokens_seen": 1136328704 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033117352056168505, + "loss": 2.7178, + "theoretical_loss": 3.6055713847944, + "tokens_seen": 1136394240 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033116349047141423, + "loss": 2.8013, + "theoretical_loss": 3.605552062019906, + "tokens_seen": 1136459776 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003311534603811434, + "loss": 2.8232, + "theoretical_loss": 3.6055327406716406, + "tokens_seen": 1136525312 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033114343029087265, + "loss": 2.6199, + "theoretical_loss": 3.6055134207494177, + "tokens_seen": 1136590848 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033113340020060177, + "loss": 2.763, + "theoretical_loss": 3.605494102253048, + "tokens_seen": 1136656384 + }, + { + "epoch": 0.34, + "learning_rate": 0.000331123370110331, + "loss": 2.7053, + "theoretical_loss": 3.6054747851823454, + "tokens_seen": 1136721920 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033111334002006013, + "loss": 2.8259, + "theoretical_loss": 3.605455469537122, + "tokens_seen": 1136787456 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033110330992978937, + "loss": 2.8541, + "theoretical_loss": 3.6054361553171903, + "tokens_seen": 1136852992 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033109327983951855, + "loss": 2.7364, + "theoretical_loss": 3.605416842522364, + "tokens_seen": 1136918528 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033108324974924773, + "loss": 2.7473, + "theoretical_loss": 3.605397531152454, + "tokens_seen": 1136984064 + }, + { + "epoch": 0.34, + "objective/train/docs_used": 631187, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.756531000137329, + "objective/train/theoretical_loss": 3.6053782212072747, + "objective/train/tokens_used": 1157509600, + "theoretical_loss": 3.6053782212072747, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033107321965897697, + "loss": 2.6511, + "theoretical_loss": 3.6053782212072747, + "tokens_seen": 1137049600 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033106318956870615, + "loss": 2.4748, + "theoretical_loss": 3.605358912686638, + "tokens_seen": 1137115136 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033105315947843533, + "loss": 2.869, + "theoretical_loss": 3.6053396055903564, + "tokens_seen": 1137180672 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310431293881645, + "loss": 2.6319, + "theoretical_loss": 3.605320299918244, + "tokens_seen": 1137246208 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310330992978937, + "loss": 2.7541, + "theoretical_loss": 3.6053009956701128, + "tokens_seen": 1137311744 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310230692076229, + "loss": 2.8495, + "theoretical_loss": 3.605281692845776, + "tokens_seen": 1137377280 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003310130391173521, + "loss": 2.7188, + "theoretical_loss": 3.605262391445047, + "tokens_seen": 1137442816 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033100300902708124, + "loss": 2.6702, + "theoretical_loss": 3.605243091467737, + "tokens_seen": 1137508352 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033099297893681047, + "loss": 2.6905, + "theoretical_loss": 3.605223792913661, + "tokens_seen": 1137573888 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309829488465396, + "loss": 2.9471, + "theoretical_loss": 3.6052044957826315, + "tokens_seen": 1137639424 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033097291875626883, + "loss": 2.9353, + "theoretical_loss": 3.6051852000744606, + "tokens_seen": 1137704960 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330962888665998, + "loss": 2.6811, + "theoretical_loss": 3.6051659057889633, + "tokens_seen": 1137770496 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309528585757272, + "loss": 2.8414, + "theoretical_loss": 3.6051466129259513, + "tokens_seen": 1137836032 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309428284854564, + "loss": 2.5861, + "theoretical_loss": 3.6051273214852384, + "tokens_seen": 1137901568 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309327983951856, + "loss": 2.8635, + "theoretical_loss": 3.6051080314666377, + "tokens_seen": 1137967104 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033092276830491474, + "loss": 2.6327, + "theoretical_loss": 3.6050887428699623, + "tokens_seen": 1138032640 + }, + { + "epoch": 0.34, + "learning_rate": 0.000330912738214644, + "loss": 2.5742, + "theoretical_loss": 3.6050694556950256, + "tokens_seen": 1138098176 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003309027081243731, + "loss": 2.8405, + "theoretical_loss": 3.6050501699416415, + "tokens_seen": 1138163712 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033089267803410234, + "loss": 2.8262, + "theoretical_loss": 3.6050308856096223, + "tokens_seen": 1138229248 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308826479438315, + "loss": 2.6865, + "theoretical_loss": 3.6050116026987826, + "tokens_seen": 1138294784 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308726178535607, + "loss": 2.6635, + "theoretical_loss": 3.6049923212089343, + "tokens_seen": 1138360320 + }, + { + "epoch": 0.34, + "learning_rate": 0.0003308625877632899, + "loss": 2.8313, + "theoretical_loss": 3.6049730411398926, + "tokens_seen": 1138425856 + }, + { + "epoch": 0.34, + "learning_rate": 0.00033085255767301906, + "loss": 2.6591, + "theoretical_loss": 3.60495376249147, + "tokens_seen": 1138491392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033084252758274824, + "loss": 2.6289, + "theoretical_loss": 3.604934485263481, + "tokens_seen": 1138556928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003308324974924775, + "loss": 2.7447, + "theoretical_loss": 3.604915209455738, + "tokens_seen": 1138622464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 631834, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4626145362854004, + "objective/train/theoretical_loss": 3.604895935068055, + "objective/train/tokens_used": 1159148000, + "theoretical_loss": 3.604895935068055, + "tokens_seen": 1138688000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003308224674022066, + "loss": 2.8697, + "theoretical_loss": 3.604895935068055, + "tokens_seen": 1138688000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033081243731193584, + "loss": 2.8243, + "theoretical_loss": 3.604876662100246, + "tokens_seen": 1138753536 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033080240722166497, + "loss": 2.742, + "theoretical_loss": 3.604857390552125, + "tokens_seen": 1138819072 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307923771313942, + "loss": 2.7061, + "theoretical_loss": 3.604838120423504, + "tokens_seen": 1138884608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307823470411234, + "loss": 2.7016, + "theoretical_loss": 3.6048188517141995, + "tokens_seen": 1138950144 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033077231695085256, + "loss": 2.7084, + "theoretical_loss": 3.604799584424023, + "tokens_seen": 1139015680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033076228686058175, + "loss": 2.7386, + "theoretical_loss": 3.604780318552789, + "tokens_seen": 1139081216 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330752256770311, + "loss": 2.7143, + "theoretical_loss": 3.6047610541003117, + "tokens_seen": 1139146752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307422266800401, + "loss": 2.8785, + "theoretical_loss": 3.604741791066405, + "tokens_seen": 1139212288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033073219658976934, + "loss": 2.7387, + "theoretical_loss": 3.6047225294508825, + "tokens_seen": 1139277824 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033072216649949847, + "loss": 2.6078, + "theoretical_loss": 3.6047032692535588, + "tokens_seen": 1139343360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307121364092277, + "loss": 2.6801, + "theoretical_loss": 3.6046840104742466, + "tokens_seen": 1139408896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003307021063189569, + "loss": 2.6139, + "theoretical_loss": 3.6046647531127616, + "tokens_seen": 1139474432 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033069207622868607, + "loss": 2.7409, + "theoretical_loss": 3.604645497168917, + "tokens_seen": 1139539968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033068204613841525, + "loss": 2.6384, + "theoretical_loss": 3.6046262426425266, + "tokens_seen": 1139605504 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033067201604814443, + "loss": 2.7318, + "theoretical_loss": 3.604606989533405, + "tokens_seen": 1139671040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306619859578736, + "loss": 2.9143, + "theoretical_loss": 3.6045877378413667, + "tokens_seen": 1139736576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033065195586760285, + "loss": 2.6698, + "theoretical_loss": 3.604568487566225, + "tokens_seen": 1139802112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033064192577733197, + "loss": 2.7636, + "theoretical_loss": 3.6045492387077958, + "tokens_seen": 1139867648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003306318956870612, + "loss": 2.7425, + "theoretical_loss": 3.604529991265892, + "tokens_seen": 1139933184 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033062186559679034, + "loss": 2.602, + "theoretical_loss": 3.6045107452403276, + "tokens_seen": 1139998720 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033061183550651957, + "loss": 2.7949, + "theoretical_loss": 3.6044915006309184, + "tokens_seen": 1140064256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033060180541624875, + "loss": 2.6797, + "theoretical_loss": 3.6044722574374775, + "tokens_seen": 1140129792 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033059177532597793, + "loss": 2.5068, + "theoretical_loss": 3.60445301565982, + "tokens_seen": 1140195328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305817452357071, + "loss": 2.5564, + "theoretical_loss": 3.6044337752977604, + "tokens_seen": 1140260864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 632825, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7563414573669434, + "objective/train/theoretical_loss": 3.604414536351113, + "objective/train/tokens_used": 1160786400, + "theoretical_loss": 3.604414536351113, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033057171514543635, + "loss": 2.8835, + "theoretical_loss": 3.604414536351113, + "tokens_seen": 1140326400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305616850551655, + "loss": 2.8897, + "theoretical_loss": 3.604395298819692, + "tokens_seen": 1140391936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305516549648947, + "loss": 2.8563, + "theoretical_loss": 3.604376062703313, + "tokens_seen": 1140457472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033054162487462384, + "loss": 2.7353, + "theoretical_loss": 3.60435682800179, + "tokens_seen": 1140523008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305315947843531, + "loss": 2.7761, + "theoretical_loss": 3.604337594714937, + "tokens_seen": 1140588544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033052156469408225, + "loss": 2.6665, + "theoretical_loss": 3.6043183628425695, + "tokens_seen": 1140654080 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033051153460381144, + "loss": 2.8084, + "theoretical_loss": 3.6042991323845017, + "tokens_seen": 1140719616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003305015045135406, + "loss": 2.91, + "theoretical_loss": 3.604279903340549, + "tokens_seen": 1140785152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304914744232698, + "loss": 2.5918, + "theoretical_loss": 3.6042606757105258, + "tokens_seen": 1140850688 + }, + { + "epoch": 0.35, + "learning_rate": 0.000330481444332999, + "loss": 2.7085, + "theoretical_loss": 3.604241449494247, + "tokens_seen": 1140916224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304714142427282, + "loss": 2.6838, + "theoretical_loss": 3.6042222246915276, + "tokens_seen": 1140981760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033046138415245734, + "loss": 2.591, + "theoretical_loss": 3.604203001302182, + "tokens_seen": 1141047296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304513540621866, + "loss": 2.6663, + "theoretical_loss": 3.604183779326026, + "tokens_seen": 1141112832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304413239719157, + "loss": 2.9393, + "theoretical_loss": 3.604164558762873, + "tokens_seen": 1141178368 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033043129388164494, + "loss": 2.6918, + "theoretical_loss": 3.6041453396125394, + "tokens_seen": 1141243904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304212637913741, + "loss": 2.6888, + "theoretical_loss": 3.60412612187484, + "tokens_seen": 1141309440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304112337011033, + "loss": 2.7761, + "theoretical_loss": 3.604106905549589, + "tokens_seen": 1141374976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003304012036108325, + "loss": 2.8362, + "theoretical_loss": 3.604087690636603, + "tokens_seen": 1141440512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303911735205617, + "loss": 2.6173, + "theoretical_loss": 3.6040684771356957, + "tokens_seen": 1141506048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033038114343029084, + "loss": 2.7558, + "theoretical_loss": 3.604049265046683, + "tokens_seen": 1141571584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303711133400201, + "loss": 2.8015, + "theoretical_loss": 3.60403005436938, + "tokens_seen": 1141637120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303610832497492, + "loss": 2.7054, + "theoretical_loss": 3.604010845103602, + "tokens_seen": 1141702656 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033035105315947844, + "loss": 2.8058, + "theoretical_loss": 3.6039916372491643, + "tokens_seen": 1141768192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303410230692076, + "loss": 2.8196, + "theoretical_loss": 3.6039724308058814, + "tokens_seen": 1141833728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303309929789368, + "loss": 2.581, + "theoretical_loss": 3.60395322577357, + "tokens_seen": 1141899264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 633465, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.615844488143921, + "objective/train/theoretical_loss": 3.6039340221520444, + "objective/train/tokens_used": 1162424800, + "theoretical_loss": 3.6039340221520444, + "tokens_seen": 1141964800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033032096288866604, + "loss": 2.6319, + "theoretical_loss": 3.6039340221520444, + "tokens_seen": 1141964800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033031093279839517, + "loss": 2.7764, + "theoretical_loss": 3.6039148199411204, + "tokens_seen": 1142030336 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003303009027081244, + "loss": 2.768, + "theoretical_loss": 3.6038956191406135, + "tokens_seen": 1142095872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302908726178536, + "loss": 2.7994, + "theoretical_loss": 3.6038764197503395, + "tokens_seen": 1142161408 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033028084252758276, + "loss": 2.818, + "theoretical_loss": 3.603857221770113, + "tokens_seen": 1142226944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033027081243731195, + "loss": 2.7977, + "theoretical_loss": 3.6038380251997504, + "tokens_seen": 1142292480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302607823470412, + "loss": 2.7323, + "theoretical_loss": 3.603818830039067, + "tokens_seen": 1142358016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302507522567703, + "loss": 2.6949, + "theoretical_loss": 3.603799636287878, + "tokens_seen": 1142423552 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033024072216649954, + "loss": 2.7476, + "theoretical_loss": 3.6037804439459995, + "tokens_seen": 1142489088 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033023069207622867, + "loss": 2.8042, + "theoretical_loss": 3.6037612530132477, + "tokens_seen": 1142554624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302206619859579, + "loss": 2.832, + "theoretical_loss": 3.6037420634894373, + "tokens_seen": 1142620160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003302106318956871, + "loss": 2.5317, + "theoretical_loss": 3.6037228753743844, + "tokens_seen": 1142685696 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033020060180541627, + "loss": 2.791, + "theoretical_loss": 3.6037036886679052, + "tokens_seen": 1142751232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033019057171514545, + "loss": 2.543, + "theoretical_loss": 3.603684503369815, + "tokens_seen": 1142816768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033018054162487463, + "loss": 2.9402, + "theoretical_loss": 3.60366531947993, + "tokens_seen": 1142882304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301705115346038, + "loss": 2.9351, + "theoretical_loss": 3.6036461369980657, + "tokens_seen": 1142947840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033016048144433305, + "loss": 2.765, + "theoretical_loss": 3.603626955924039, + "tokens_seen": 1143013376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301504513540622, + "loss": 2.6821, + "theoretical_loss": 3.6036077762576646, + "tokens_seen": 1143078912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003301404212637914, + "loss": 2.6081, + "theoretical_loss": 3.603588597998759, + "tokens_seen": 1143144448 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033013039117352054, + "loss": 2.854, + "theoretical_loss": 3.6035694211471387, + "tokens_seen": 1143209984 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033012036108324977, + "loss": 2.6444, + "theoretical_loss": 3.6035502457026194, + "tokens_seen": 1143275520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033011033099297895, + "loss": 2.5662, + "theoretical_loss": 3.603531071665017, + "tokens_seen": 1143341056 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033010030090270813, + "loss": 2.7041, + "theoretical_loss": 3.6035118990341477, + "tokens_seen": 1143406592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300902708124373, + "loss": 2.7002, + "theoretical_loss": 3.6034927278098277, + "tokens_seen": 1143472128 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033008024072216655, + "loss": 2.7704, + "theoretical_loss": 3.6034735579918737, + "tokens_seen": 1143537664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 634749, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3306100368499756, + "objective/train/theoretical_loss": 3.6034543895801017, + "objective/train/tokens_used": 1164063200, + "theoretical_loss": 3.6034543895801017, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300702106318957, + "loss": 2.654, + "theoretical_loss": 3.6034543895801017, + "tokens_seen": 1143603200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300601805416249, + "loss": 2.7415, + "theoretical_loss": 3.603435222574327, + "tokens_seen": 1143668736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033005015045135404, + "loss": 2.6088, + "theoretical_loss": 3.6034160569743676, + "tokens_seen": 1143734272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300401203610833, + "loss": 2.5878, + "theoretical_loss": 3.6033968927800384, + "tokens_seen": 1143799808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033003009027081245, + "loss": 2.7632, + "theoretical_loss": 3.6033777299911565, + "tokens_seen": 1143865344 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033002006018054164, + "loss": 2.9358, + "theoretical_loss": 3.6033585686075384, + "tokens_seen": 1143930880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003300100300902708, + "loss": 2.7065, + "theoretical_loss": 3.603339408629, + "tokens_seen": 1143996416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00033, + "loss": 2.7274, + "theoretical_loss": 3.6033202500553587, + "tokens_seen": 1144061952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299899699097292, + "loss": 2.6696, + "theoretical_loss": 3.6033010928864297, + "tokens_seen": 1144127488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299799398194584, + "loss": 2.6557, + "theoretical_loss": 3.6032819371220306, + "tokens_seen": 1144193024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032996990972918754, + "loss": 2.838, + "theoretical_loss": 3.603262782761978, + "tokens_seen": 1144258560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299598796389168, + "loss": 2.8272, + "theoretical_loss": 3.6032436298060873, + "tokens_seen": 1144324096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299498495486459, + "loss": 2.7172, + "theoretical_loss": 3.6032244782541767, + "tokens_seen": 1144389632 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032993981945837514, + "loss": 2.7073, + "theoretical_loss": 3.6032053281060623, + "tokens_seen": 1144455168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299297893681043, + "loss": 2.7657, + "theoretical_loss": 3.6031861793615603, + "tokens_seen": 1144520704 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299197592778335, + "loss": 2.8131, + "theoretical_loss": 3.6031670320204885, + "tokens_seen": 1144586240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003299097291875627, + "loss": 2.7006, + "theoretical_loss": 3.6031478860826627, + "tokens_seen": 1144651776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298996990972919, + "loss": 2.7931, + "theoretical_loss": 3.6031287415479003, + "tokens_seen": 1144717312 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032988966900702104, + "loss": 2.7996, + "theoretical_loss": 3.603109598416018, + "tokens_seen": 1144782848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298796389167503, + "loss": 2.4818, + "theoretical_loss": 3.6030904566868323, + "tokens_seen": 1144848384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298696088264794, + "loss": 2.9172, + "theoretical_loss": 3.603071316360161, + "tokens_seen": 1144913920 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032985957873620864, + "loss": 2.7094, + "theoretical_loss": 3.6030521774358206, + "tokens_seen": 1144979456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298495486459378, + "loss": 2.7017, + "theoretical_loss": 3.603033039913628, + "tokens_seen": 1145044992 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329839518555667, + "loss": 2.7211, + "theoretical_loss": 3.6030139037934, + "tokens_seen": 1145110528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003298294884653962, + "loss": 2.6337, + "theoretical_loss": 3.602994769074954, + "tokens_seen": 1145176064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 635305, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9339799880981445, + "objective/train/theoretical_loss": 3.6029756357581073, + "objective/train/tokens_used": 1165701600, + "theoretical_loss": 3.6029756357581073, + "tokens_seen": 1145241600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032981945837512537, + "loss": 2.6328, + "theoretical_loss": 3.6029756357581073, + "tokens_seen": 1145241600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032980942828485455, + "loss": 2.7999, + "theoretical_loss": 3.6029565038426767, + "tokens_seen": 1145307136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297993981945838, + "loss": 2.8183, + "theoretical_loss": 3.60293737332848, + "tokens_seen": 1145372672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297893681043129, + "loss": 2.8277, + "theoretical_loss": 3.602918244215333, + "tokens_seen": 1145438208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032977933801404215, + "loss": 2.6376, + "theoretical_loss": 3.602899116503055, + "tokens_seen": 1145503744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297693079237713, + "loss": 2.7554, + "theoretical_loss": 3.6028799901914614, + "tokens_seen": 1145569280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297592778335005, + "loss": 2.8153, + "theoretical_loss": 3.6028608652803706, + "tokens_seen": 1145634816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297492477432297, + "loss": 2.4666, + "theoretical_loss": 3.6028417417695993, + "tokens_seen": 1145700352 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032973921765295887, + "loss": 2.726, + "theoretical_loss": 3.6028226196589648, + "tokens_seen": 1145765888 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032972918756268805, + "loss": 2.6854, + "theoretical_loss": 3.6028034989482856, + "tokens_seen": 1145831424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297191574724173, + "loss": 2.5511, + "theoretical_loss": 3.602784379637378, + "tokens_seen": 1145896960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003297091273821464, + "loss": 2.547, + "theoretical_loss": 3.60276526172606, + "tokens_seen": 1145962496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032969909729187565, + "loss": 2.7832, + "theoretical_loss": 3.602746145214149, + "tokens_seen": 1146028032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296890672016048, + "loss": 2.6622, + "theoretical_loss": 3.6027270301014624, + "tokens_seen": 1146093568 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329679037111334, + "loss": 2.6686, + "theoretical_loss": 3.602707916387818, + "tokens_seen": 1146159104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296690070210632, + "loss": 2.7567, + "theoretical_loss": 3.6026888040730336, + "tokens_seen": 1146224640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296589769307924, + "loss": 2.6816, + "theoretical_loss": 3.602669693156926, + "tokens_seen": 1146290176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032964894684052155, + "loss": 2.829, + "theoretical_loss": 3.6026505836393143, + "tokens_seen": 1146355712 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032963891675025074, + "loss": 2.7428, + "theoretical_loss": 3.602631475520015, + "tokens_seen": 1146421248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296288866599799, + "loss": 2.6902, + "theoretical_loss": 3.6026123687988463, + "tokens_seen": 1146486784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032961885656970915, + "loss": 2.7686, + "theoretical_loss": 3.602593263475626, + "tokens_seen": 1146552320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003296088264794383, + "loss": 2.7541, + "theoretical_loss": 3.602574159550172, + "tokens_seen": 1146617856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295987963891675, + "loss": 2.4678, + "theoretical_loss": 3.6025550570223013, + "tokens_seen": 1146683392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295887662988967, + "loss": 2.6954, + "theoretical_loss": 3.6025359558918333, + "tokens_seen": 1146748928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295787362086259, + "loss": 2.6374, + "theoretical_loss": 3.6025168561585845, + "tokens_seen": 1146814464 + }, + { + "debugging/Self-BLEU-5": 0.4786247835242382, + "debugging/distinct-1-grams": 0.7609861614906261, + "debugging/distinct-2-grams": 0.9220814316505062, + "debugging/entropy-1-grams": 5.880984152522802, + "debugging/entropy-2-grams": 6.5464310105420465, + "debugging/length": 573.6, + "debugging/num_segments": 10, + "epoch": 0.35, + "objective/train/docs_used": 636617, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.868852376937866, + "objective/train/theoretical_loss": 3.6024977578223742, + "objective/train/tokens_used": 1167340000, + "theoretical_loss": 3.6024977578223742, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295687061183551, + "loss": 2.8676, + "theoretical_loss": 3.6024977578223742, + "tokens_seen": 1146880000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032955867602808424, + "loss": 2.6701, + "theoretical_loss": 3.6024786608830195, + "tokens_seen": 1146945536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295486459378135, + "loss": 2.9224, + "theoretical_loss": 3.6024595653403386, + "tokens_seen": 1147011072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032953861584754265, + "loss": 2.8055, + "theoretical_loss": 3.602440471194149, + "tokens_seen": 1147076608 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032952858575727184, + "loss": 2.686, + "theoretical_loss": 3.6024213784442702, + "tokens_seen": 1147142144 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329518555667001, + "loss": 2.8341, + "theoretical_loss": 3.602402287090519, + "tokens_seen": 1147207680 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003295085255767302, + "loss": 2.8485, + "theoretical_loss": 3.6023831971327143, + "tokens_seen": 1147273216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294984954864594, + "loss": 2.6909, + "theoretical_loss": 3.602364108570674, + "tokens_seen": 1147338752 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294884653961886, + "loss": 2.7537, + "theoretical_loss": 3.6023450214042163, + "tokens_seen": 1147404288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032947843530591774, + "loss": 2.8729, + "theoretical_loss": 3.60232593563316, + "tokens_seen": 1147469824 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329468405215647, + "loss": 2.9974, + "theoretical_loss": 3.602306851257322, + "tokens_seen": 1147535360 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294583751253761, + "loss": 2.6533, + "theoretical_loss": 3.602287768276523, + "tokens_seen": 1147600896 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032944834503510534, + "loss": 2.793, + "theoretical_loss": 3.6022686866905786, + "tokens_seen": 1147666432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294383149448345, + "loss": 2.6353, + "theoretical_loss": 3.602249606499309, + "tokens_seen": 1147731968 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294282848545637, + "loss": 2.8417, + "theoretical_loss": 3.602230527702533, + "tokens_seen": 1147797504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294182547642929, + "loss": 2.7079, + "theoretical_loss": 3.602211450300067, + "tokens_seen": 1147863040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003294082246740221, + "loss": 2.8198, + "theoretical_loss": 3.6021923742917314, + "tokens_seen": 1147928576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032939819458375124, + "loss": 2.6095, + "theoretical_loss": 3.602173299677344, + "tokens_seen": 1147994112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293881644934805, + "loss": 2.7788, + "theoretical_loss": 3.6021542264567237, + "tokens_seen": 1148059648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293781344032096, + "loss": 2.7369, + "theoretical_loss": 3.6021351546296887, + "tokens_seen": 1148125184 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032936810431293884, + "loss": 2.6818, + "theoretical_loss": 3.6021160841960578, + "tokens_seen": 1148190720 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329358074222668, + "loss": 2.8319, + "theoretical_loss": 3.6020970151556497, + "tokens_seen": 1148256256 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293480441323972, + "loss": 2.6298, + "theoretical_loss": 3.602077947508283, + "tokens_seen": 1148321792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003293380140421264, + "loss": 2.7902, + "theoretical_loss": 3.602058881253777, + "tokens_seen": 1148387328 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032932798395185557, + "loss": 2.7109, + "theoretical_loss": 3.6020398163919496, + "tokens_seen": 1148452864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 637752, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6599230766296387, + "objective/train/theoretical_loss": 3.6020207529226203, + "objective/train/tokens_used": 1168978400, + "theoretical_loss": 3.6020207529226203, + "tokens_seen": 1148518400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032931795386158475, + "loss": 2.7556, + "theoretical_loss": 3.6020207529226203, + "tokens_seen": 1148518400 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329307923771314, + "loss": 2.7028, + "theoretical_loss": 3.6020016908456074, + "tokens_seen": 1148583936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292978936810431, + "loss": 2.6435, + "theoretical_loss": 3.6019826301607303, + "tokens_seen": 1148649472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032928786359077235, + "loss": 2.7049, + "theoretical_loss": 3.6019635708678077, + "tokens_seen": 1148715008 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292778335005015, + "loss": 2.7472, + "theoretical_loss": 3.601944512966658, + "tokens_seen": 1148780544 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292678034102307, + "loss": 2.7465, + "theoretical_loss": 3.601925456457101, + "tokens_seen": 1148846080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292577733199599, + "loss": 2.5858, + "theoretical_loss": 3.6019064013389555, + "tokens_seen": 1148911616 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032924774322968907, + "loss": 2.8702, + "theoretical_loss": 3.60188734761204, + "tokens_seen": 1148977152 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032923771313941825, + "loss": 2.6107, + "theoretical_loss": 3.6018682952761747, + "tokens_seen": 1149042688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292276830491475, + "loss": 2.8954, + "theoretical_loss": 3.6018492443311776, + "tokens_seen": 1149108224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003292176529588766, + "loss": 2.5675, + "theoretical_loss": 3.6018301947768685, + "tokens_seen": 1149173760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032920762286860585, + "loss": 2.7175, + "theoretical_loss": 3.6018111466130662, + "tokens_seen": 1149239296 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329197592778335, + "loss": 2.7522, + "theoretical_loss": 3.6017920998395896, + "tokens_seen": 1149304832 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291875626880642, + "loss": 2.6172, + "theoretical_loss": 3.6017730544562596, + "tokens_seen": 1149370368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291775325977934, + "loss": 2.6186, + "theoretical_loss": 3.6017540104628933, + "tokens_seen": 1149435904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291675025075226, + "loss": 2.8122, + "theoretical_loss": 3.6017349678593114, + "tokens_seen": 1149501440 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032915747241725175, + "loss": 2.5775, + "theoretical_loss": 3.601715926645333, + "tokens_seen": 1149566976 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032914744232698094, + "loss": 2.7746, + "theoretical_loss": 3.6016968868207773, + "tokens_seen": 1149632512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291374122367101, + "loss": 2.5421, + "theoretical_loss": 3.6016778483854637, + "tokens_seen": 1149698048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032912738214643935, + "loss": 2.4038, + "theoretical_loss": 3.6016588113392114, + "tokens_seen": 1149763584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291173520561685, + "loss": 2.7402, + "theoretical_loss": 3.601639775681841, + "tokens_seen": 1149829120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003291073219658977, + "loss": 2.7332, + "theoretical_loss": 3.6016207414131705, + "tokens_seen": 1149894656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290972918756269, + "loss": 2.7302, + "theoretical_loss": 3.6016017085330203, + "tokens_seen": 1149960192 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290872617853561, + "loss": 2.661, + "theoretical_loss": 3.6015826770412103, + "tokens_seen": 1150025728 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032907723169508526, + "loss": 2.7305, + "theoretical_loss": 3.601563646937559, + "tokens_seen": 1150091264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 638347, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3350415229797363, + "objective/train/theoretical_loss": 3.6015446182218875, + "objective/train/tokens_used": 1170616800, + "theoretical_loss": 3.6015446182218875, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032906720160481444, + "loss": 2.6234, + "theoretical_loss": 3.6015446182218875, + "tokens_seen": 1150156800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290571715145436, + "loss": 2.5445, + "theoretical_loss": 3.6015255908940147, + "tokens_seen": 1150222336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032904714142427286, + "loss": 2.6305, + "theoretical_loss": 3.60150656495376, + "tokens_seen": 1150287872 + }, + { + "epoch": 0.35, + "learning_rate": 0.000329037111334002, + "loss": 2.7704, + "theoretical_loss": 3.6014875404009437, + "tokens_seen": 1150353408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290270812437312, + "loss": 2.8285, + "theoretical_loss": 3.601468517235385, + "tokens_seen": 1150418944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032901705115346034, + "loss": 2.7081, + "theoretical_loss": 3.6014494954569045, + "tokens_seen": 1150484480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003290070210631896, + "loss": 2.7436, + "theoretical_loss": 3.601430475065322, + "tokens_seen": 1150550016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032899699097291876, + "loss": 2.7384, + "theoretical_loss": 3.6014114560604567, + "tokens_seen": 1150615552 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032898696088264794, + "loss": 2.7074, + "theoretical_loss": 3.6013924384421285, + "tokens_seen": 1150681088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289769307923771, + "loss": 2.671, + "theoretical_loss": 3.6013734222101585, + "tokens_seen": 1150746624 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289669007021063, + "loss": 2.6452, + "theoretical_loss": 3.6013544073643655, + "tokens_seen": 1150812160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289568706118355, + "loss": 2.5962, + "theoretical_loss": 3.60133539390457, + "tokens_seen": 1150877696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289468405215647, + "loss": 2.8025, + "theoretical_loss": 3.601316381830592, + "tokens_seen": 1150943232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032893681043129385, + "loss": 2.6795, + "theoretical_loss": 3.601297371142252, + "tokens_seen": 1151008768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003289267803410231, + "loss": 2.7709, + "theoretical_loss": 3.6012783618393693, + "tokens_seen": 1151074304 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032891675025075226, + "loss": 2.7108, + "theoretical_loss": 3.601259353921765, + "tokens_seen": 1151139840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032890672016048145, + "loss": 2.8169, + "theoretical_loss": 3.6012403473892585, + "tokens_seen": 1151205376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288966900702106, + "loss": 2.7462, + "theoretical_loss": 3.6012213422416703, + "tokens_seen": 1151270912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288866599799398, + "loss": 2.6478, + "theoretical_loss": 3.6012023384788208, + "tokens_seen": 1151336448 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328876629889669, + "loss": 2.7033, + "theoretical_loss": 3.60118333610053, + "tokens_seen": 1151401984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288665997993982, + "loss": 2.7022, + "theoretical_loss": 3.6011643351066183, + "tokens_seen": 1151467520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032885656970912735, + "loss": 2.8157, + "theoretical_loss": 3.6011453354969065, + "tokens_seen": 1151533056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288465396188566, + "loss": 2.6348, + "theoretical_loss": 3.6011263372712143, + "tokens_seen": 1151598592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288365095285857, + "loss": 2.7876, + "theoretical_loss": 3.6011073404293628, + "tokens_seen": 1151664128 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032882647943831495, + "loss": 2.5848, + "theoretical_loss": 3.601088344971172, + "tokens_seen": 1151729664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 639523, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5807628631591797, + "objective/train/theoretical_loss": 3.6010693508964624, + "objective/train/tokens_used": 1172255200, + "theoretical_loss": 3.6010693508964624, + "tokens_seen": 1151795200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288164493480442, + "loss": 2.5903, + "theoretical_loss": 3.6010693508964624, + "tokens_seen": 1151795200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003288064192577733, + "loss": 2.5482, + "theoretical_loss": 3.601050358205055, + "tokens_seen": 1151860736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032879638916750255, + "loss": 2.7664, + "theoretical_loss": 3.6010313668967697, + "tokens_seen": 1151926272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287863590772317, + "loss": 2.7388, + "theoretical_loss": 3.6010123769714273, + "tokens_seen": 1151991808 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287763289869609, + "loss": 2.7822, + "theoretical_loss": 3.600993388428849, + "tokens_seen": 1152057344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287662988966901, + "loss": 2.8795, + "theoretical_loss": 3.600974401268855, + "tokens_seen": 1152122880 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032875626880641927, + "loss": 2.5308, + "theoretical_loss": 3.600955415491266, + "tokens_seen": 1152188416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032874623871614845, + "loss": 2.9124, + "theoretical_loss": 3.6009364310959024, + "tokens_seen": 1152253952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287362086258777, + "loss": 2.7283, + "theoretical_loss": 3.6009174480825856, + "tokens_seen": 1152319488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287261785356068, + "loss": 2.7157, + "theoretical_loss": 3.6008984664511363, + "tokens_seen": 1152385024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032871614844533605, + "loss": 2.5741, + "theoretical_loss": 3.6008794862013747, + "tokens_seen": 1152450560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003287061183550652, + "loss": 2.9161, + "theoretical_loss": 3.600860507333122, + "tokens_seen": 1152516096 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286960882647944, + "loss": 2.6876, + "theoretical_loss": 3.6008415298461993, + "tokens_seen": 1152581632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286860581745236, + "loss": 2.7743, + "theoretical_loss": 3.6008225537404277, + "tokens_seen": 1152647168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286760280842528, + "loss": 2.8304, + "theoretical_loss": 3.6008035790156274, + "tokens_seen": 1152712704 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032866599799398195, + "loss": 2.9008, + "theoretical_loss": 3.6007846056716204, + "tokens_seen": 1152778240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032865596790371114, + "loss": 2.7073, + "theoretical_loss": 3.6007656337082268, + "tokens_seen": 1152843776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286459378134403, + "loss": 2.4973, + "theoretical_loss": 3.6007466631252685, + "tokens_seen": 1152909312 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032863590772316955, + "loss": 2.6311, + "theoretical_loss": 3.6007276939225656, + "tokens_seen": 1152974848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286258776328987, + "loss": 2.5766, + "theoretical_loss": 3.60070872609994, + "tokens_seen": 1153040384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286158475426279, + "loss": 2.5913, + "theoretical_loss": 3.6006897596572127, + "tokens_seen": 1153105920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003286058174523571, + "loss": 2.5173, + "theoretical_loss": 3.6006707945942047, + "tokens_seen": 1153171456 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285957873620863, + "loss": 2.6512, + "theoretical_loss": 3.6006518309107376, + "tokens_seen": 1153236992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032858575727181546, + "loss": 2.8246, + "theoretical_loss": 3.6006328686066325, + "tokens_seen": 1153302528 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032857572718154464, + "loss": 2.8009, + "theoretical_loss": 3.6006139076817103, + "tokens_seen": 1153368064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 639983, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.798460006713867, + "objective/train/theoretical_loss": 3.600594948135793, + "objective/train/tokens_used": 1173893600, + "theoretical_loss": 3.600594948135793, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285656970912738, + "loss": 2.6424, + "theoretical_loss": 3.600594948135793, + "tokens_seen": 1153433600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032855566700100306, + "loss": 2.6186, + "theoretical_loss": 3.6005759899687018, + "tokens_seen": 1153499136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285456369107322, + "loss": 2.8315, + "theoretical_loss": 3.6005570331802574, + "tokens_seen": 1153564672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285356068204614, + "loss": 2.7475, + "theoretical_loss": 3.600538077770282, + "tokens_seen": 1153630208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032852557673019054, + "loss": 2.7912, + "theoretical_loss": 3.6005191237385965, + "tokens_seen": 1153695744 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003285155466399198, + "loss": 2.6463, + "theoretical_loss": 3.6005001710850233, + "tokens_seen": 1153761280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032850551654964896, + "loss": 2.5891, + "theoretical_loss": 3.6004812198093834, + "tokens_seen": 1153826816 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032849548645937814, + "loss": 2.4599, + "theoretical_loss": 3.600462269911498, + "tokens_seen": 1153892352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284854563691073, + "loss": 2.6544, + "theoretical_loss": 3.6004433213911886, + "tokens_seen": 1153957888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284754262788365, + "loss": 2.5024, + "theoretical_loss": 3.6004243742482775, + "tokens_seen": 1154023424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284653961885657, + "loss": 2.5078, + "theoretical_loss": 3.600405428482586, + "tokens_seen": 1154088960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284553660982949, + "loss": 2.6429, + "theoretical_loss": 3.600386484093936, + "tokens_seen": 1154154496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032844533600802405, + "loss": 2.6753, + "theoretical_loss": 3.600367541082149, + "tokens_seen": 1154220032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284353059177533, + "loss": 2.6927, + "theoretical_loss": 3.6003485994470474, + "tokens_seen": 1154285568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032842527582748246, + "loss": 2.5926, + "theoretical_loss": 3.600329659188452, + "tokens_seen": 1154351104 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032841524573721165, + "loss": 2.6645, + "theoretical_loss": 3.6003107203061853, + "tokens_seen": 1154416640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003284052156469408, + "loss": 2.8488, + "theoretical_loss": 3.6002917828000687, + "tokens_seen": 1154482176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032839518555667, + "loss": 2.6092, + "theoretical_loss": 3.6002728466699248, + "tokens_seen": 1154547712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283851554663992, + "loss": 2.67, + "theoretical_loss": 3.6002539119155745, + "tokens_seen": 1154613248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283751253761284, + "loss": 2.57, + "theoretical_loss": 3.6002349785368413, + "tokens_seen": 1154678784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032836509528585755, + "loss": 2.594, + "theoretical_loss": 3.6002160465335455, + "tokens_seen": 1154744320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283550651955868, + "loss": 2.8139, + "theoretical_loss": 3.6001971159055097, + "tokens_seen": 1154809856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283450351053159, + "loss": 2.6035, + "theoretical_loss": 3.600178186652556, + "tokens_seen": 1154875392 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032833500501504515, + "loss": 2.8044, + "theoretical_loss": 3.6001592587745073, + "tokens_seen": 1154940928 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032832497492477433, + "loss": 2.6125, + "theoretical_loss": 3.6001403322711845, + "tokens_seen": 1155006464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 641200, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4352283477783203, + "objective/train/theoretical_loss": 3.6001214071424106, + "objective/train/tokens_used": 1175532000, + "theoretical_loss": 3.6001214071424106, + "tokens_seen": 1155072000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283149448345035, + "loss": 2.718, + "theoretical_loss": 3.6001214071424106, + "tokens_seen": 1155072000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003283049147442327, + "loss": 2.7494, + "theoretical_loss": 3.6001024833880075, + "tokens_seen": 1155137536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282948846539619, + "loss": 2.7, + "theoretical_loss": 3.6000835610077973, + "tokens_seen": 1155203072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032828485456369105, + "loss": 2.6363, + "theoretical_loss": 3.6000646400016025, + "tokens_seen": 1155268608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282748244734203, + "loss": 2.6263, + "theoretical_loss": 3.6000457203692453, + "tokens_seen": 1155334144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282647943831494, + "loss": 2.5974, + "theoretical_loss": 3.6000268021105475, + "tokens_seen": 1155399680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032825476429287865, + "loss": 2.9712, + "theoretical_loss": 3.6000078852253328, + "tokens_seen": 1155465216 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032824473420260783, + "loss": 2.5518, + "theoretical_loss": 3.599988969713422, + "tokens_seen": 1155530752 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328234704112337, + "loss": 2.6989, + "theoretical_loss": 3.5999700555746386, + "tokens_seen": 1155596288 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282246740220662, + "loss": 2.7771, + "theoretical_loss": 3.5999511428088047, + "tokens_seen": 1155661824 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003282146439317954, + "loss": 2.6778, + "theoretical_loss": 3.5999322314157425, + "tokens_seen": 1155727360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032820461384152456, + "loss": 2.5627, + "theoretical_loss": 3.5999133213952756, + "tokens_seen": 1155792896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281945837512538, + "loss": 2.9634, + "theoretical_loss": 3.599894412747225, + "tokens_seen": 1155858432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281845536609829, + "loss": 2.8353, + "theoretical_loss": 3.599875505471415, + "tokens_seen": 1155923968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032817452357071215, + "loss": 2.9044, + "theoretical_loss": 3.599856599567666, + "tokens_seen": 1155989504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281644934804413, + "loss": 2.6862, + "theoretical_loss": 3.599837695035803, + "tokens_seen": 1156055040 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281544633901705, + "loss": 2.8285, + "theoretical_loss": 3.5998187918756472, + "tokens_seen": 1156120576 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281444332998997, + "loss": 2.8731, + "theoretical_loss": 3.599799890087022, + "tokens_seen": 1156186112 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281344032096289, + "loss": 2.6179, + "theoretical_loss": 3.5997809896697497, + "tokens_seen": 1156251648 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032812437311935806, + "loss": 2.5605, + "theoretical_loss": 3.5997620906236536, + "tokens_seen": 1156317184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281143430290873, + "loss": 2.6217, + "theoretical_loss": 3.599743192948556, + "tokens_seen": 1156382720 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003281043129388164, + "loss": 2.7596, + "theoretical_loss": 3.59972429664428, + "tokens_seen": 1156448256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032809428284854566, + "loss": 2.5259, + "theoretical_loss": 3.599705401710648, + "tokens_seen": 1156513792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280842527582748, + "loss": 2.473, + "theoretical_loss": 3.5996865081474843, + "tokens_seen": 1156579328 + }, + { + "epoch": 0.35, + "learning_rate": 0.000328074222668004, + "loss": 2.6603, + "theoretical_loss": 3.59966761595461, + "tokens_seen": 1156644864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 641801, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.768510341644287, + "objective/train/theoretical_loss": 3.59964872513185, + "objective/train/tokens_used": 1177170400, + "theoretical_loss": 3.59964872513185, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032806419257773326, + "loss": 2.7684, + "theoretical_loss": 3.59964872513185, + "tokens_seen": 1156710400 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280541624874624, + "loss": 2.7661, + "theoretical_loss": 3.5996298356790257, + "tokens_seen": 1156775936 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003280441323971916, + "loss": 2.5801, + "theoretical_loss": 3.599610947595961, + "tokens_seen": 1156841472 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032803410230692074, + "loss": 2.8947, + "theoretical_loss": 3.599592060882479, + "tokens_seen": 1156907008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032802407221665, + "loss": 2.8118, + "theoretical_loss": 3.599573175538402, + "tokens_seen": 1156972544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032801404212637916, + "loss": 2.713, + "theoretical_loss": 3.599554291563554, + "tokens_seen": 1157038080 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032800401203610834, + "loss": 2.6654, + "theoretical_loss": 3.5995354089577583, + "tokens_seen": 1157103616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279939819458375, + "loss": 2.4662, + "theoretical_loss": 3.599516527720837, + "tokens_seen": 1157169152 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279839518555667, + "loss": 2.7589, + "theoretical_loss": 3.5994976478526146, + "tokens_seen": 1157234688 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279739217652959, + "loss": 2.7968, + "theoretical_loss": 3.599478769352914, + "tokens_seen": 1157300224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279638916750251, + "loss": 2.5783, + "theoretical_loss": 3.5994598922215584, + "tokens_seen": 1157365760 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032795386158475425, + "loss": 2.662, + "theoretical_loss": 3.5994410164583703, + "tokens_seen": 1157431296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279438314944835, + "loss": 2.5359, + "theoretical_loss": 3.5994221420631747, + "tokens_seen": 1157496832 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032793380140421266, + "loss": 2.7496, + "theoretical_loss": 3.5994032690357938, + "tokens_seen": 1157562368 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032792377131394185, + "loss": 2.7392, + "theoretical_loss": 3.5993843973760518, + "tokens_seen": 1157627904 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327913741223671, + "loss": 2.6497, + "theoretical_loss": 3.599365527083772, + "tokens_seen": 1157693440 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003279037111334002, + "loss": 2.6444, + "theoretical_loss": 3.599346658158777, + "tokens_seen": 1157758976 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278936810431294, + "loss": 2.8481, + "theoretical_loss": 3.599327790600891, + "tokens_seen": 1157824512 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278836509528586, + "loss": 2.7322, + "theoretical_loss": 3.5993089244099385, + "tokens_seen": 1157890048 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032787362086258775, + "loss": 2.6314, + "theoretical_loss": 3.5992900595857416, + "tokens_seen": 1157955584 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327863590772317, + "loss": 2.5972, + "theoretical_loss": 3.599271196128125, + "tokens_seen": 1158021120 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278535606820461, + "loss": 2.6548, + "theoretical_loss": 3.599252334036912, + "tokens_seen": 1158086656 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032784353059177535, + "loss": 2.5857, + "theoretical_loss": 3.5992334733119264, + "tokens_seen": 1158152192 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032783350050150453, + "loss": 2.7087, + "theoretical_loss": 3.5992146139529915, + "tokens_seen": 1158217728 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278234704112337, + "loss": 2.5285, + "theoretical_loss": 3.599195755959931, + "tokens_seen": 1158283264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 642800, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7663416862487793, + "objective/train/theoretical_loss": 3.59917689933257, + "objective/train/tokens_used": 1178808800, + "theoretical_loss": 3.59917689933257, + "tokens_seen": 1158348800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003278134403209629, + "loss": 2.7091, + "theoretical_loss": 3.59917689933257, + "tokens_seen": 1158348800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032780341023069213, + "loss": 2.6744, + "theoretical_loss": 3.599158044070731, + "tokens_seen": 1158414336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032779338014042125, + "loss": 2.5622, + "theoretical_loss": 3.5991391901742382, + "tokens_seen": 1158479872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277833500501505, + "loss": 2.6717, + "theoretical_loss": 3.599120337642916, + "tokens_seen": 1158545408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277733199598796, + "loss": 2.5785, + "theoretical_loss": 3.599101486476587, + "tokens_seen": 1158610944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032776328986960885, + "loss": 2.5764, + "theoretical_loss": 3.599082636675077, + "tokens_seen": 1158676480 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032775325977933803, + "loss": 2.6596, + "theoretical_loss": 3.599063788238209, + "tokens_seen": 1158742016 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277432296890672, + "loss": 2.6874, + "theoretical_loss": 3.599044941165807, + "tokens_seen": 1158807552 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277331995987964, + "loss": 2.8803, + "theoretical_loss": 3.5990260954576954, + "tokens_seen": 1158873088 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003277231695085256, + "loss": 2.793, + "theoretical_loss": 3.599007251113698, + "tokens_seen": 1158938624 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032771313941825476, + "loss": 2.7827, + "theoretical_loss": 3.598988408133639, + "tokens_seen": 1159004160 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327703109327984, + "loss": 2.7677, + "theoretical_loss": 3.5989695665173427, + "tokens_seen": 1159069696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276930792377131, + "loss": 2.5865, + "theoretical_loss": 3.598950726264633, + "tokens_seen": 1159135232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032768304914744235, + "loss": 2.6556, + "theoretical_loss": 3.5989318873753344, + "tokens_seen": 1159200768 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276730190571715, + "loss": 2.8157, + "theoretical_loss": 3.5989130498492714, + "tokens_seen": 1159266304 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276629889669007, + "loss": 2.6851, + "theoretical_loss": 3.5988942136862674, + "tokens_seen": 1159331840 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276529588766299, + "loss": 2.7396, + "theoretical_loss": 3.598875378886148, + "tokens_seen": 1159397376 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276429287863591, + "loss": 2.8136, + "theoretical_loss": 3.5988565454487365, + "tokens_seen": 1159462912 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032763289869608826, + "loss": 2.8422, + "theoretical_loss": 3.598837713373858, + "tokens_seen": 1159528448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276228686058175, + "loss": 2.6567, + "theoretical_loss": 3.598818882661336, + "tokens_seen": 1159593984 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003276128385155466, + "loss": 2.694, + "theoretical_loss": 3.598800053310996, + "tokens_seen": 1159659520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032760280842527586, + "loss": 2.7281, + "theoretical_loss": 3.5987812253226616, + "tokens_seen": 1159725056 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327592778335005, + "loss": 2.7379, + "theoretical_loss": 3.598762398696158, + "tokens_seen": 1159790592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275827482447342, + "loss": 2.8922, + "theoretical_loss": 3.5987435734313094, + "tokens_seen": 1159856128 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275727181544634, + "loss": 2.5817, + "theoretical_loss": 3.598724749527941, + "tokens_seen": 1159921664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 643907, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6200287342071533, + "objective/train/theoretical_loss": 3.5987059269858763, + "objective/train/tokens_used": 1180447200, + "theoretical_loss": 3.5987059269858763, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275626880641926, + "loss": 2.7223, + "theoretical_loss": 3.5987059269858763, + "tokens_seen": 1159987200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032755265797392176, + "loss": 2.7649, + "theoretical_loss": 3.5986871058049408, + "tokens_seen": 1160052736 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032754262788365094, + "loss": 2.5639, + "theoretical_loss": 3.598668285984959, + "tokens_seen": 1160118272 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275325977933801, + "loss": 2.7138, + "theoretical_loss": 3.5986494675257554, + "tokens_seen": 1160183808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032752256770310936, + "loss": 2.6548, + "theoretical_loss": 3.598630650427155, + "tokens_seen": 1160249344 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275125376128385, + "loss": 2.6848, + "theoretical_loss": 3.598611834688983, + "tokens_seen": 1160314880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003275025075225677, + "loss": 2.7612, + "theoretical_loss": 3.5985930203110628, + "tokens_seen": 1160380416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032749247743229685, + "loss": 2.9665, + "theoretical_loss": 3.598574207293221, + "tokens_seen": 1160445952 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274824473420261, + "loss": 2.8244, + "theoretical_loss": 3.598555395635281, + "tokens_seen": 1160511488 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032747241725175527, + "loss": 2.5306, + "theoretical_loss": 3.5985365853370688, + "tokens_seen": 1160577024 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032746238716148445, + "loss": 2.7923, + "theoretical_loss": 3.5985177763984084, + "tokens_seen": 1160642560 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032745235707121363, + "loss": 2.756, + "theoretical_loss": 3.5984989688191256, + "tokens_seen": 1160708096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032744232698094286, + "loss": 2.6654, + "theoretical_loss": 3.598480162599045, + "tokens_seen": 1160773632 + }, + { + "epoch": 0.35, + "learning_rate": 0.000327432296890672, + "loss": 2.8767, + "theoretical_loss": 3.5984613577379916, + "tokens_seen": 1160839168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274222668004012, + "loss": 2.6347, + "theoretical_loss": 3.598442554235791, + "tokens_seen": 1160904704 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032741223671013035, + "loss": 2.6026, + "theoretical_loss": 3.598423752092268, + "tokens_seen": 1160970240 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003274022066198596, + "loss": 2.5634, + "theoretical_loss": 3.5984049513072467, + "tokens_seen": 1161035776 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032739217652958877, + "loss": 2.6826, + "theoretical_loss": 3.598386151880554, + "tokens_seen": 1161101312 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032738214643931795, + "loss": 2.8304, + "theoretical_loss": 3.598367353812014, + "tokens_seen": 1161166848 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032737211634904713, + "loss": 2.6202, + "theoretical_loss": 3.5983485571014526, + "tokens_seen": 1161232384 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273620862587763, + "loss": 2.5217, + "theoretical_loss": 3.5983297617486945, + "tokens_seen": 1161297920 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273520561685055, + "loss": 2.6864, + "theoretical_loss": 3.598310967753565, + "tokens_seen": 1161363456 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032734202607823473, + "loss": 2.776, + "theoretical_loss": 3.59829217511589, + "tokens_seen": 1161428992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032733199598796386, + "loss": 2.6287, + "theoretical_loss": 3.5982733838354948, + "tokens_seen": 1161494528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003273219658976931, + "loss": 2.5982, + "theoretical_loss": 3.5982545939122037, + "tokens_seen": 1161560064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 644495, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7944414615631104, + "objective/train/theoretical_loss": 3.598235805345843, + "objective/train/tokens_used": 1182085600, + "theoretical_loss": 3.598235805345843, + "tokens_seen": 1161625600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032731193580742233, + "loss": 2.7485, + "theoretical_loss": 3.598235805345843, + "tokens_seen": 1161625600 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032730190571715145, + "loss": 2.7042, + "theoretical_loss": 3.598217018136239, + "tokens_seen": 1161691136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272918756268807, + "loss": 2.6849, + "theoretical_loss": 3.598198232283216, + "tokens_seen": 1161756672 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272818455366098, + "loss": 2.5112, + "theoretical_loss": 3.598179447786599, + "tokens_seen": 1161822208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032727181544633905, + "loss": 2.5599, + "theoretical_loss": 3.598160664646215, + "tokens_seen": 1161887744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032726178535606823, + "loss": 2.4779, + "theoretical_loss": 3.598141882861889, + "tokens_seen": 1161953280 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272517552657974, + "loss": 2.508, + "theoretical_loss": 3.598123102433447, + "tokens_seen": 1162018816 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272417251755266, + "loss": 2.7165, + "theoretical_loss": 3.5981043233607135, + "tokens_seen": 1162084352 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272316950852558, + "loss": 2.6167, + "theoretical_loss": 3.5980855456435155, + "tokens_seen": 1162149888 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032722166499498496, + "loss": 2.6884, + "theoretical_loss": 3.5980667692816777, + "tokens_seen": 1162215424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272116349047142, + "loss": 2.5378, + "theoretical_loss": 3.598047994275026, + "tokens_seen": 1162280960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003272016048144433, + "loss": 2.6135, + "theoretical_loss": 3.5980292206233875, + "tokens_seen": 1162346496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032719157472417256, + "loss": 2.6741, + "theoretical_loss": 3.598010448326586, + "tokens_seen": 1162412032 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271815446339017, + "loss": 2.7771, + "theoretical_loss": 3.5979916773844494, + "tokens_seen": 1162477568 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271715145436309, + "loss": 2.5643, + "theoretical_loss": 3.5979729077968017, + "tokens_seen": 1162543104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271614844533601, + "loss": 2.6286, + "theoretical_loss": 3.5979541395634698, + "tokens_seen": 1162608640 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271514543630893, + "loss": 2.7594, + "theoretical_loss": 3.59793537268428, + "tokens_seen": 1162674176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032714142427281846, + "loss": 2.5682, + "theoretical_loss": 3.5979166071590574, + "tokens_seen": 1162739712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271313941825477, + "loss": 2.8591, + "theoretical_loss": 3.5978978429876283, + "tokens_seen": 1162805248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271213640922768, + "loss": 2.5559, + "theoretical_loss": 3.597879080169819, + "tokens_seen": 1162870784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032711133400200606, + "loss": 2.6989, + "theoretical_loss": 3.5978603187054556, + "tokens_seen": 1162936320 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003271013039117352, + "loss": 2.4975, + "theoretical_loss": 3.5978415585943635, + "tokens_seen": 1163001856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270912738214644, + "loss": 2.7157, + "theoretical_loss": 3.5978227998363694, + "tokens_seen": 1163067392 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270812437311936, + "loss": 2.5283, + "theoretical_loss": 3.5978040424313003, + "tokens_seen": 1163132928 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270712136409228, + "loss": 2.7726, + "theoretical_loss": 3.5977852863789805, + "tokens_seen": 1163198464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 645795, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8706414699554443, + "objective/train/theoretical_loss": 3.5977665316792375, + "objective/train/tokens_used": 1183724000, + "theoretical_loss": 3.5977665316792375, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032706118355065196, + "loss": 2.708, + "theoretical_loss": 3.5977665316792375, + "tokens_seen": 1163264000 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032705115346038114, + "loss": 2.54, + "theoretical_loss": 3.5977477783318976, + "tokens_seen": 1163329536 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270411233701103, + "loss": 2.5949, + "theoretical_loss": 3.597729026336787, + "tokens_seen": 1163395072 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032703109327983956, + "loss": 2.7173, + "theoretical_loss": 3.597710275693731, + "tokens_seen": 1163460608 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270210631895687, + "loss": 2.8881, + "theoretical_loss": 3.5976915264025577, + "tokens_seen": 1163526144 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003270110330992979, + "loss": 2.7011, + "theoretical_loss": 3.5976727784630924, + "tokens_seen": 1163591680 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032700100300902705, + "loss": 2.7247, + "theoretical_loss": 3.5976540318751615, + "tokens_seen": 1163657216 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269909729187563, + "loss": 2.6947, + "theoretical_loss": 3.597635286638592, + "tokens_seen": 1163722752 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032698094282848547, + "loss": 2.7205, + "theoretical_loss": 3.59761654275321, + "tokens_seen": 1163788288 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032697091273821465, + "loss": 2.4699, + "theoretical_loss": 3.597597800218842, + "tokens_seen": 1163853824 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032696088264794383, + "loss": 2.5679, + "theoretical_loss": 3.597579059035314, + "tokens_seen": 1163919360 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032695085255767306, + "loss": 2.6229, + "theoretical_loss": 3.5975603192024543, + "tokens_seen": 1163984896 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269408224674022, + "loss": 2.8093, + "theoretical_loss": 3.5975415807200886, + "tokens_seen": 1164050432 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269307923771314, + "loss": 2.6794, + "theoretical_loss": 3.5975228435880426, + "tokens_seen": 1164115968 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032692076228686055, + "loss": 2.6717, + "theoretical_loss": 3.5975041078061443, + "tokens_seen": 1164181504 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003269107321965898, + "loss": 2.6432, + "theoretical_loss": 3.5974853733742194, + "tokens_seen": 1164247040 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032690070210631897, + "loss": 2.4695, + "theoretical_loss": 3.5974666402920956, + "tokens_seen": 1164312576 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032689067201604815, + "loss": 2.6219, + "theoretical_loss": 3.5974479085596, + "tokens_seen": 1164378112 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032688064192577733, + "loss": 2.7599, + "theoretical_loss": 3.5974291781765575, + "tokens_seen": 1164443648 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268706118355065, + "loss": 2.849, + "theoretical_loss": 3.5974104491427963, + "tokens_seen": 1164509184 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268605817452357, + "loss": 2.7382, + "theoretical_loss": 3.5973917214581435, + "tokens_seen": 1164574720 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032685055165496493, + "loss": 2.6985, + "theoretical_loss": 3.5973729951224254, + "tokens_seen": 1164640256 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032684052156469406, + "loss": 2.619, + "theoretical_loss": 3.597354270135469, + "tokens_seen": 1164705792 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268304914744233, + "loss": 2.4724, + "theoretical_loss": 3.5973355464971015, + "tokens_seen": 1164771328 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003268204613841524, + "loss": 2.5872, + "theoretical_loss": 3.59731682420715, + "tokens_seen": 1164836864 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 647021, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5365681648254395, + "objective/train/theoretical_loss": 3.597298103265441, + "objective/train/tokens_used": 1185362400, + "theoretical_loss": 3.597298103265441, + "tokens_seen": 1164902400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032681043129388165, + "loss": 2.7167, + "theoretical_loss": 3.597298103265441, + "tokens_seen": 1164902400 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032680040120361084, + "loss": 2.7747, + "theoretical_loss": 3.597279383671802, + "tokens_seen": 1164967936 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032679037111334, + "loss": 2.7435, + "theoretical_loss": 3.5972606654260604, + "tokens_seen": 1165033472 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267803410230692, + "loss": 2.5863, + "theoretical_loss": 3.5972419485280422, + "tokens_seen": 1165099008 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032677031093279843, + "loss": 2.6398, + "theoretical_loss": 3.597223232977576, + "tokens_seen": 1165164544 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032676028084252756, + "loss": 2.77, + "theoretical_loss": 3.597204518774488, + "tokens_seen": 1165230080 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267502507522568, + "loss": 2.7466, + "theoretical_loss": 3.5971858059186057, + "tokens_seen": 1165295616 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267402206619859, + "loss": 2.5286, + "theoretical_loss": 3.5971670944097562, + "tokens_seen": 1165361152 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032673019057171516, + "loss": 2.6559, + "theoretical_loss": 3.5971483842477676, + "tokens_seen": 1165426688 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032672016048144434, + "loss": 2.7506, + "theoretical_loss": 3.597129675432466, + "tokens_seen": 1165492224 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267101303911735, + "loss": 2.6339, + "theoretical_loss": 3.5971109679636797, + "tokens_seen": 1165557760 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003267001003009027, + "loss": 2.6564, + "theoretical_loss": 3.5970922618412353, + "tokens_seen": 1165623296 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266900702106319, + "loss": 2.6649, + "theoretical_loss": 3.5970735570649612, + "tokens_seen": 1165688832 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032668004012036106, + "loss": 2.5365, + "theoretical_loss": 3.597054853634684, + "tokens_seen": 1165754368 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266700100300903, + "loss": 2.5501, + "theoretical_loss": 3.597036151550231, + "tokens_seen": 1165819904 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266599799398194, + "loss": 2.8168, + "theoretical_loss": 3.597017450811431, + "tokens_seen": 1165885440 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032664994984954866, + "loss": 2.511, + "theoretical_loss": 3.5969987514181105, + "tokens_seen": 1165950976 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032663991975927784, + "loss": 2.8327, + "theoretical_loss": 3.596980053370097, + "tokens_seen": 1166016512 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326629889669007, + "loss": 2.77, + "theoretical_loss": 3.5969613566672187, + "tokens_seen": 1166082048 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266198595787362, + "loss": 2.8883, + "theoretical_loss": 3.596942661309303, + "tokens_seen": 1166147584 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003266098294884654, + "loss": 2.9158, + "theoretical_loss": 3.5969239672961777, + "tokens_seen": 1166213120 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032659979939819457, + "loss": 2.8605, + "theoretical_loss": 3.5969052746276704, + "tokens_seen": 1166278656 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265897693079238, + "loss": 2.8674, + "theoretical_loss": 3.596886583303608, + "tokens_seen": 1166344192 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032657973921765293, + "loss": 2.4217, + "theoretical_loss": 3.59686789332382, + "tokens_seen": 1166409728 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032656970912738216, + "loss": 2.7101, + "theoretical_loss": 3.596849204688133, + "tokens_seen": 1166475264 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 647459, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1023616790771484, + "objective/train/theoretical_loss": 3.5968305173963744, + "objective/train/tokens_used": 1187000800, + "theoretical_loss": 3.5968305173963744, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032655967903711135, + "loss": 2.8593, + "theoretical_loss": 3.5968305173963744, + "tokens_seen": 1166540800 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265496489468405, + "loss": 2.5972, + "theoretical_loss": 3.596811831448373, + "tokens_seen": 1166606336 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032653961885656976, + "loss": 2.7343, + "theoretical_loss": 3.596793146843957, + "tokens_seen": 1166671872 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265295887662989, + "loss": 2.5819, + "theoretical_loss": 3.5967744635829533, + "tokens_seen": 1166737408 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003265195586760281, + "loss": 2.665, + "theoretical_loss": 3.5967557816651903, + "tokens_seen": 1166802944 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032650952858575725, + "loss": 2.6375, + "theoretical_loss": 3.596737101090496, + "tokens_seen": 1166868480 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264994984954865, + "loss": 2.5396, + "theoretical_loss": 3.5967184218586983, + "tokens_seen": 1166934016 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032648946840521567, + "loss": 2.6808, + "theoretical_loss": 3.5966997439696256, + "tokens_seen": 1166999552 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032647943831494485, + "loss": 2.6276, + "theoretical_loss": 3.596681067423106, + "tokens_seen": 1167065088 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032646940822467403, + "loss": 2.5045, + "theoretical_loss": 3.596662392218967, + "tokens_seen": 1167130624 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032645937813440326, + "loss": 2.4189, + "theoretical_loss": 3.5966437183570372, + "tokens_seen": 1167196160 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264493480441324, + "loss": 2.7956, + "theoretical_loss": 3.596625045837145, + "tokens_seen": 1167261696 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003264393179538616, + "loss": 2.559, + "theoretical_loss": 3.5966063746591184, + "tokens_seen": 1167327232 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032642928786359075, + "loss": 2.7304, + "theoretical_loss": 3.596587704822785, + "tokens_seen": 1167392768 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032641925777332, + "loss": 2.6686, + "theoretical_loss": 3.596569036327974, + "tokens_seen": 1167458304 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032640922768304917, + "loss": 2.476, + "theoretical_loss": 3.596550369174513, + "tokens_seen": 1167523840 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032639919759277835, + "loss": 2.5304, + "theoretical_loss": 3.596531703362231, + "tokens_seen": 1167589376 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032638916750250753, + "loss": 2.7702, + "theoretical_loss": 3.596513038890956, + "tokens_seen": 1167654912 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263791374122367, + "loss": 2.6531, + "theoretical_loss": 3.596494375760516, + "tokens_seen": 1167720448 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263691073219659, + "loss": 2.7071, + "theoretical_loss": 3.59647571397074, + "tokens_seen": 1167785984 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032635907723169513, + "loss": 2.8008, + "theoretical_loss": 3.596457053521457, + "tokens_seen": 1167851520 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032634904714142426, + "loss": 2.6172, + "theoretical_loss": 3.5964383944124947, + "tokens_seen": 1167917056 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263390170511535, + "loss": 2.8027, + "theoretical_loss": 3.596419736643681, + "tokens_seen": 1167982592 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003263289869608826, + "loss": 2.5849, + "theoretical_loss": 3.5964010802148456, + "tokens_seen": 1168048128 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032631895687061185, + "loss": 2.8076, + "theoretical_loss": 3.596382425125817, + "tokens_seen": 1168113664 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 648684, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9439427852630615, + "objective/train/theoretical_loss": 3.5963637713764234, + "objective/train/tokens_used": 1188639200, + "theoretical_loss": 3.5963637713764234, + "tokens_seen": 1168179200 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032630892678034104, + "loss": 2.6827, + "theoretical_loss": 3.5963637713764234, + "tokens_seen": 1168179200 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262988966900702, + "loss": 2.7814, + "theoretical_loss": 3.5963451189664934, + "tokens_seen": 1168244736 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262888665997994, + "loss": 2.6662, + "theoretical_loss": 3.596326467895856, + "tokens_seen": 1168310272 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032627883650952863, + "loss": 2.5774, + "theoretical_loss": 3.5963078181643398, + "tokens_seen": 1168375808 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032626880641925776, + "loss": 2.7241, + "theoretical_loss": 3.5962891697717736, + "tokens_seen": 1168441344 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326258776328987, + "loss": 2.6469, + "theoretical_loss": 3.596270522717986, + "tokens_seen": 1168506880 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262487462387161, + "loss": 2.5015, + "theoretical_loss": 3.596251877002806, + "tokens_seen": 1168572416 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032623871614844536, + "loss": 2.8421, + "theoretical_loss": 3.596233232626062, + "tokens_seen": 1168637952 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032622868605817454, + "loss": 2.7734, + "theoretical_loss": 3.596214589587584, + "tokens_seen": 1168703488 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262186559679037, + "loss": 2.7135, + "theoretical_loss": 3.5961959478871997, + "tokens_seen": 1168769024 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003262086258776329, + "loss": 2.5319, + "theoretical_loss": 3.596177307524738, + "tokens_seen": 1168834560 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261985957873621, + "loss": 2.6577, + "theoretical_loss": 3.5961586685000295, + "tokens_seen": 1168900096 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032618856569709126, + "loss": 2.4889, + "theoretical_loss": 3.596140030812901, + "tokens_seen": 1168965632 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261785356068205, + "loss": 2.5488, + "theoretical_loss": 3.596121394463183, + "tokens_seen": 1169031168 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261685055165496, + "loss": 2.6289, + "theoretical_loss": 3.5961027594507042, + "tokens_seen": 1169096704 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032615847542627886, + "loss": 2.5204, + "theoretical_loss": 3.5960841257752936, + "tokens_seen": 1169162240 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032614844533600804, + "loss": 2.8722, + "theoretical_loss": 3.5960654934367806, + "tokens_seen": 1169227776 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261384152457372, + "loss": 2.8192, + "theoretical_loss": 3.5960468624349935, + "tokens_seen": 1169293312 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261283851554664, + "loss": 2.7604, + "theoretical_loss": 3.596028232769762, + "tokens_seen": 1169358848 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003261183550651956, + "loss": 2.5521, + "theoretical_loss": 3.5960096044409164, + "tokens_seen": 1169424384 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032610832497492477, + "loss": 2.8361, + "theoretical_loss": 3.595990977448284, + "tokens_seen": 1169489920 + }, + { + "epoch": 0.35, + "learning_rate": 0.000326098294884654, + "loss": 2.7227, + "theoretical_loss": 3.595972351791696, + "tokens_seen": 1169555456 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032608826479438313, + "loss": 2.698, + "theoretical_loss": 3.59595372747098, + "tokens_seen": 1169620992 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032607823470411236, + "loss": 2.6011, + "theoretical_loss": 3.5959351044859664, + "tokens_seen": 1169686528 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260682046138415, + "loss": 2.7583, + "theoretical_loss": 3.5959164828364845, + "tokens_seen": 1169752064 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 649144, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.792980670928955, + "objective/train/theoretical_loss": 3.5958978625223628, + "objective/train/tokens_used": 1190277600, + "theoretical_loss": 3.5958978625223628, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260581745235707, + "loss": 2.5967, + "theoretical_loss": 3.5958978625223628, + "tokens_seen": 1169817600 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260481444332999, + "loss": 2.7829, + "theoretical_loss": 3.595879243543432, + "tokens_seen": 1169883136 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003260381143430291, + "loss": 2.6743, + "theoretical_loss": 3.5958606258995207, + "tokens_seen": 1169948672 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032602808425275827, + "loss": 2.5841, + "theoretical_loss": 3.5958420095904593, + "tokens_seen": 1170014208 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032601805416248745, + "loss": 2.8274, + "theoretical_loss": 3.5958233946160765, + "tokens_seen": 1170079744 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032600802407221663, + "loss": 2.6443, + "theoretical_loss": 3.595804780976202, + "tokens_seen": 1170145280 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032599799398194587, + "loss": 2.3263, + "theoretical_loss": 3.5957861686706654, + "tokens_seen": 1170210816 + }, + { + "epoch": 0.35, + "learning_rate": 0.000325987963891675, + "loss": 2.4542, + "theoretical_loss": 3.595767557699297, + "tokens_seen": 1170276352 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032597793380140423, + "loss": 2.7791, + "theoretical_loss": 3.595748948061925, + "tokens_seen": 1170341888 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259679037111334, + "loss": 2.6827, + "theoretical_loss": 3.59573033975838, + "tokens_seen": 1170407424 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259578736208626, + "loss": 2.6128, + "theoretical_loss": 3.5957117327884927, + "tokens_seen": 1170472960 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259478435305918, + "loss": 2.8103, + "theoretical_loss": 3.595693127152091, + "tokens_seen": 1170538496 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032593781344032095, + "loss": 2.7226, + "theoretical_loss": 3.5956745228490057, + "tokens_seen": 1170604032 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032592778335005014, + "loss": 2.8914, + "theoretical_loss": 3.5956559198790665, + "tokens_seen": 1170669568 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032591775325977937, + "loss": 2.6374, + "theoretical_loss": 3.595637318242103, + "tokens_seen": 1170735104 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003259077231695085, + "loss": 2.682, + "theoretical_loss": 3.5956187179379455, + "tokens_seen": 1170800640 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032589769307923773, + "loss": 2.6491, + "theoretical_loss": 3.595600118966424, + "tokens_seen": 1170866176 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032588766298896686, + "loss": 2.7347, + "theoretical_loss": 3.5955815213273676, + "tokens_seen": 1170931712 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258776328986961, + "loss": 2.5143, + "theoretical_loss": 3.595562925020607, + "tokens_seen": 1170997248 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258676028084253, + "loss": 2.7273, + "theoretical_loss": 3.595544330045972, + "tokens_seen": 1171062784 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032585757271815446, + "loss": 2.7069, + "theoretical_loss": 3.5955257364032924, + "tokens_seen": 1171128320 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032584754262788364, + "loss": 2.4612, + "theoretical_loss": 3.595507144092399, + "tokens_seen": 1171193856 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258375125376128, + "loss": 2.8743, + "theoretical_loss": 3.595488553113121, + "tokens_seen": 1171259392 + }, + { + "epoch": 0.35, + "learning_rate": 0.000325827482447342, + "loss": 2.6314, + "theoretical_loss": 3.5954699634652894, + "tokens_seen": 1171324928 + }, + { + "epoch": 0.35, + "learning_rate": 0.00032581745235707124, + "loss": 2.4695, + "theoretical_loss": 3.595451375148733, + "tokens_seen": 1171390464 + }, + { + "epoch": 0.35, + "objective/train/docs_used": 649993, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.923766851425171, + "objective/train/theoretical_loss": 3.5954327881632837, + "objective/train/tokens_used": 1191916000, + "theoretical_loss": 3.5954327881632837, + "tokens_seen": 1171456000 + }, + { + "epoch": 0.35, + "learning_rate": 0.0003258074222668004, + "loss": 2.8658, + "theoretical_loss": 3.5954327881632837, + "tokens_seen": 1171456000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257973921765296, + "loss": 2.7553, + "theoretical_loss": 3.5954142025087705, + "tokens_seen": 1171521536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032578736208625883, + "loss": 2.7047, + "theoretical_loss": 3.595395618185024, + "tokens_seen": 1171587072 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032577733199598796, + "loss": 3.073, + "theoretical_loss": 3.5953770351918743, + "tokens_seen": 1171652608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257673019057172, + "loss": 2.5345, + "theoretical_loss": 3.595358453529152, + "tokens_seen": 1171718144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257572718154463, + "loss": 2.5694, + "theoretical_loss": 3.595339873196688, + "tokens_seen": 1171783680 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032574724172517556, + "loss": 2.8471, + "theoretical_loss": 3.5953212941943113, + "tokens_seen": 1171849216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032573721163490474, + "loss": 2.6878, + "theoretical_loss": 3.5953027165218536, + "tokens_seen": 1171914752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257271815446339, + "loss": 2.6934, + "theoretical_loss": 3.595284140179144, + "tokens_seen": 1171980288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257171514543631, + "loss": 2.6298, + "theoretical_loss": 3.595265565166015, + "tokens_seen": 1172045824 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003257071213640923, + "loss": 2.5254, + "theoretical_loss": 3.595246991482295, + "tokens_seen": 1172111360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032569709127382146, + "loss": 2.6913, + "theoretical_loss": 3.5952284191278157, + "tokens_seen": 1172176896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256870611835507, + "loss": 2.5984, + "theoretical_loss": 3.595209848102407, + "tokens_seen": 1172242432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256770310932798, + "loss": 2.5922, + "theoretical_loss": 3.5951912784059, + "tokens_seen": 1172307968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032566700100300906, + "loss": 2.7977, + "theoretical_loss": 3.5951727100381254, + "tokens_seen": 1172373504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032565697091273824, + "loss": 2.6315, + "theoretical_loss": 3.5951541429989136, + "tokens_seen": 1172439040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256469408224674, + "loss": 3.0059, + "theoretical_loss": 3.595135577288095, + "tokens_seen": 1172504576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256369107321966, + "loss": 2.5167, + "theoretical_loss": 3.595117012905501, + "tokens_seen": 1172570112 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256268806419258, + "loss": 2.4955, + "theoretical_loss": 3.595098449850962, + "tokens_seen": 1172635648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032561685055165497, + "loss": 2.638, + "theoretical_loss": 3.5950798881243085, + "tokens_seen": 1172701184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003256068204613842, + "loss": 2.4716, + "theoretical_loss": 3.5950613277253716, + "tokens_seen": 1172766720 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032559679037111333, + "loss": 2.5471, + "theoretical_loss": 3.595042768653982, + "tokens_seen": 1172832256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032558676028084256, + "loss": 2.829, + "theoretical_loss": 3.595024210909971, + "tokens_seen": 1172897792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255767301905717, + "loss": 2.5036, + "theoretical_loss": 3.5950056544931694, + "tokens_seen": 1172963328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255667001003009, + "loss": 2.784, + "theoretical_loss": 3.594987099403407, + "tokens_seen": 1173028864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 651258, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.56889009475708, + "objective/train/theoretical_loss": 3.5949685456405165, + "objective/train/tokens_used": 1193554400, + "theoretical_loss": 3.5949685456405165, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255566700100301, + "loss": 2.6219, + "theoretical_loss": 3.5949685456405165, + "tokens_seen": 1173094400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003255466399197593, + "loss": 2.7571, + "theoretical_loss": 3.594949993204328, + "tokens_seen": 1173159936 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032553660982948847, + "loss": 2.609, + "theoretical_loss": 3.5949314420946727, + "tokens_seen": 1173225472 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032552657973921765, + "loss": 2.7424, + "theoretical_loss": 3.594912892311381, + "tokens_seen": 1173291008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032551654964894683, + "loss": 2.6436, + "theoretical_loss": 3.594894343854285, + "tokens_seen": 1173356544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032550651955867607, + "loss": 2.7312, + "theoretical_loss": 3.5948757967232154, + "tokens_seen": 1173422080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254964894684052, + "loss": 2.657, + "theoretical_loss": 3.594857250918003, + "tokens_seen": 1173487616 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032548645937813443, + "loss": 2.8031, + "theoretical_loss": 3.5948387064384795, + "tokens_seen": 1173553152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254764292878636, + "loss": 2.6133, + "theoretical_loss": 3.594820163284476, + "tokens_seen": 1173618688 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254663991975928, + "loss": 2.9284, + "theoretical_loss": 3.594801621455824, + "tokens_seen": 1173684224 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325456369107322, + "loss": 2.6838, + "theoretical_loss": 3.594783080952354, + "tokens_seen": 1173749760 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032544633901705115, + "loss": 2.5229, + "theoretical_loss": 3.5947645417738983, + "tokens_seen": 1173815296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032543630892678034, + "loss": 2.6614, + "theoretical_loss": 3.594746003920287, + "tokens_seen": 1173880832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032542627883650957, + "loss": 2.7473, + "theoretical_loss": 3.5947274673913525, + "tokens_seen": 1173946368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003254162487462387, + "loss": 2.4982, + "theoretical_loss": 3.594708932186926, + "tokens_seen": 1174011904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032540621865596793, + "loss": 2.7672, + "theoretical_loss": 3.5946903983068386, + "tokens_seen": 1174077440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032539618856569706, + "loss": 2.6318, + "theoretical_loss": 3.594671865750922, + "tokens_seen": 1174142976 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253861584754263, + "loss": 2.7851, + "theoretical_loss": 3.5946533345190073, + "tokens_seen": 1174208512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253761283851555, + "loss": 2.595, + "theoretical_loss": 3.5946348046109273, + "tokens_seen": 1174274048 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032536609829488466, + "loss": 2.7437, + "theoretical_loss": 3.5946162760265117, + "tokens_seen": 1174339584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032535606820461384, + "loss": 2.9372, + "theoretical_loss": 3.5945977487655933, + "tokens_seen": 1174405120 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325346038114343, + "loss": 2.704, + "theoretical_loss": 3.5945792228280036, + "tokens_seen": 1174470656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253360080240722, + "loss": 2.664, + "theoretical_loss": 3.5945606982135736, + "tokens_seen": 1174536192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032532597793380144, + "loss": 2.7617, + "theoretical_loss": 3.594542174922136, + "tokens_seen": 1174601728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032531594784353056, + "loss": 2.5863, + "theoretical_loss": 3.5945236529535216, + "tokens_seen": 1174667264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 652492, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.737172842025757, + "objective/train/theoretical_loss": 3.5945051323075625, + "objective/train/tokens_used": 1195192800, + "theoretical_loss": 3.5945051323075625, + "tokens_seen": 1174732800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003253059177532598, + "loss": 2.6773, + "theoretical_loss": 3.5945051323075625, + "tokens_seen": 1174732800 + }, + { + "epoch": 0.36, + "learning_rate": 0.000325295887662989, + "loss": 2.7729, + "theoretical_loss": 3.594486612984091, + "tokens_seen": 1174798336 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032528585757271816, + "loss": 2.7024, + "theoretical_loss": 3.5944680949829375, + "tokens_seen": 1174863872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032527582748244734, + "loss": 2.5224, + "theoretical_loss": 3.5944495783039354, + "tokens_seen": 1174929408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252657973921765, + "loss": 2.5866, + "theoretical_loss": 3.5944310629469154, + "tokens_seen": 1174994944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252557673019057, + "loss": 2.7831, + "theoretical_loss": 3.59441254891171, + "tokens_seen": 1175060480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032524573721163494, + "loss": 2.738, + "theoretical_loss": 3.5943940361981506, + "tokens_seen": 1175126016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032523570712136407, + "loss": 2.5638, + "theoretical_loss": 3.59437552480607, + "tokens_seen": 1175191552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003252256770310933, + "loss": 2.6825, + "theoretical_loss": 3.5943570147352997, + "tokens_seen": 1175257088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032521564694082243, + "loss": 2.6556, + "theoretical_loss": 3.594338505985671, + "tokens_seen": 1175322624 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032520561685055166, + "loss": 2.8164, + "theoretical_loss": 3.594319998557017, + "tokens_seen": 1175388160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032519558676028084, + "loss": 2.7041, + "theoretical_loss": 3.59430149244917, + "tokens_seen": 1175453696 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032518555667001, + "loss": 2.5987, + "theoretical_loss": 3.5942829876619613, + "tokens_seen": 1175519232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251755265797392, + "loss": 2.5567, + "theoretical_loss": 3.594264484195223, + "tokens_seen": 1175584768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032516549648946844, + "loss": 2.7074, + "theoretical_loss": 3.594245982048788, + "tokens_seen": 1175650304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032515546639919757, + "loss": 2.7701, + "theoretical_loss": 3.5942274812224877, + "tokens_seen": 1175715840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003251454363089268, + "loss": 2.6326, + "theoretical_loss": 3.5942089817161547, + "tokens_seen": 1175781376 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032513540621865593, + "loss": 2.608, + "theoretical_loss": 3.594190483529621, + "tokens_seen": 1175846912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032512537612838517, + "loss": 2.5178, + "theoretical_loss": 3.5941719866627193, + "tokens_seen": 1175912448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032511534603811435, + "loss": 2.737, + "theoretical_loss": 3.594153491115282, + "tokens_seen": 1175977984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032510531594784353, + "loss": 2.5786, + "theoretical_loss": 3.5941349968871408, + "tokens_seen": 1176043520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250952858575727, + "loss": 2.5329, + "theoretical_loss": 3.5941165039781287, + "tokens_seen": 1176109056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250852557673019, + "loss": 2.4487, + "theoretical_loss": 3.594098012388078, + "tokens_seen": 1176174592 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032507522567703107, + "loss": 2.5077, + "theoretical_loss": 3.5940795221168207, + "tokens_seen": 1176240128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250651955867603, + "loss": 2.5841, + "theoretical_loss": 3.59406103316419, + "tokens_seen": 1176305664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 652965, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.588792085647583, + "objective/train/theoretical_loss": 3.5940425455300176, + "objective/train/tokens_used": 1196831200, + "theoretical_loss": 3.5940425455300176, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250551654964895, + "loss": 2.6309, + "theoretical_loss": 3.5940425455300176, + "tokens_seen": 1176371200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032504513540621867, + "loss": 2.7376, + "theoretical_loss": 3.594024059214137, + "tokens_seen": 1176436736 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032503510531594785, + "loss": 2.9167, + "theoretical_loss": 3.59400557421638, + "tokens_seen": 1176502272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032502507522567703, + "loss": 2.6344, + "theoretical_loss": 3.5939870905365794, + "tokens_seen": 1176567808 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032501504513540627, + "loss": 2.6758, + "theoretical_loss": 3.593968608174568, + "tokens_seen": 1176633344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003250050150451354, + "loss": 2.6595, + "theoretical_loss": 3.593950127130178, + "tokens_seen": 1176698880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032499498495486463, + "loss": 2.6477, + "theoretical_loss": 3.5939316474032426, + "tokens_seen": 1176764416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249849548645938, + "loss": 2.6468, + "theoretical_loss": 3.593913168993594, + "tokens_seen": 1176829952 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324974924774323, + "loss": 2.7797, + "theoretical_loss": 3.5938946919010655, + "tokens_seen": 1176895488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249648946840522, + "loss": 2.6107, + "theoretical_loss": 3.59387621612549, + "tokens_seen": 1176961024 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032495486459378135, + "loss": 2.6228, + "theoretical_loss": 3.593857741666699, + "tokens_seen": 1177026560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032494483450351054, + "loss": 2.5313, + "theoretical_loss": 3.593839268524527, + "tokens_seen": 1177092096 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032493480441323977, + "loss": 2.5962, + "theoretical_loss": 3.593820796698806, + "tokens_seen": 1177157632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003249247743229689, + "loss": 2.7831, + "theoretical_loss": 3.593802326189369, + "tokens_seen": 1177223168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032491474423269813, + "loss": 2.6414, + "theoretical_loss": 3.5937838569960494, + "tokens_seen": 1177288704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032490471414242726, + "loss": 2.8084, + "theoretical_loss": 3.5937653891186794, + "tokens_seen": 1177354240 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248946840521565, + "loss": 2.7652, + "theoretical_loss": 3.5937469225570924, + "tokens_seen": 1177419776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248846539618857, + "loss": 2.8432, + "theoretical_loss": 3.593728457311122, + "tokens_seen": 1177485312 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032487462387161486, + "loss": 2.6533, + "theoretical_loss": 3.5937099933805996, + "tokens_seen": 1177550848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032486459378134404, + "loss": 2.6939, + "theoretical_loss": 3.59369153076536, + "tokens_seen": 1177616384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248545636910732, + "loss": 2.4783, + "theoretical_loss": 3.5936730694652352, + "tokens_seen": 1177681920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248445336008024, + "loss": 2.8857, + "theoretical_loss": 3.593654609480059, + "tokens_seen": 1177747456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032483450351053164, + "loss": 2.8108, + "theoretical_loss": 3.5936361508096644, + "tokens_seen": 1177812992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032482447342026076, + "loss": 2.7793, + "theoretical_loss": 3.5936176934538846, + "tokens_seen": 1177878528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032481444332999, + "loss": 2.6711, + "theoretical_loss": 3.593599237412553, + "tokens_seen": 1177944064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 652965, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3456740379333496, + "objective/train/theoretical_loss": 3.5935807826855024, + "objective/train/tokens_used": 1198469600, + "theoretical_loss": 3.5935807826855024, + "tokens_seen": 1178009600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003248044132397192, + "loss": 2.476, + "theoretical_loss": 3.5935807826855024, + "tokens_seen": 1178009600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032479438314944836, + "loss": 2.7488, + "theoretical_loss": 3.593562329272567, + "tokens_seen": 1178075136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032478435305917754, + "loss": 2.6266, + "theoretical_loss": 3.5935438771735786, + "tokens_seen": 1178140672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247743229689067, + "loss": 2.5096, + "theoretical_loss": 3.593525426388372, + "tokens_seen": 1178206208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247642928786359, + "loss": 2.8226, + "theoretical_loss": 3.59350697691678, + "tokens_seen": 1178271744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032475426278836514, + "loss": 2.5758, + "theoretical_loss": 3.593488528758636, + "tokens_seen": 1178337280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032474423269809427, + "loss": 2.7812, + "theoretical_loss": 3.5934700819137735, + "tokens_seen": 1178402816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003247342026078235, + "loss": 2.5558, + "theoretical_loss": 3.593451636382026, + "tokens_seen": 1178468352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032472417251755263, + "loss": 2.8785, + "theoretical_loss": 3.5934331921632268, + "tokens_seen": 1178533888 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032471414242728186, + "loss": 2.5582, + "theoretical_loss": 3.59341474925721, + "tokens_seen": 1178599424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032470411233701104, + "loss": 2.4771, + "theoretical_loss": 3.593396307663809, + "tokens_seen": 1178664960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246940822467402, + "loss": 2.7665, + "theoretical_loss": 3.593377867382857, + "tokens_seen": 1178730496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246840521564694, + "loss": 2.6074, + "theoretical_loss": 3.593359428414188, + "tokens_seen": 1178796032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032467402206619864, + "loss": 2.727, + "theoretical_loss": 3.5933409907576355, + "tokens_seen": 1178861568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032466399197592777, + "loss": 2.8406, + "theoretical_loss": 3.593322554413033, + "tokens_seen": 1178927104 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324653961885657, + "loss": 2.5648, + "theoretical_loss": 3.593304119380215, + "tokens_seen": 1178992640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032464393179538613, + "loss": 2.9887, + "theoretical_loss": 3.593285685659014, + "tokens_seen": 1179058176 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032463390170511537, + "loss": 2.558, + "theoretical_loss": 3.5932672532492647, + "tokens_seen": 1179123712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032462387161484455, + "loss": 2.4303, + "theoretical_loss": 3.5932488221508008, + "tokens_seen": 1179189248 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032461384152457373, + "loss": 2.68, + "theoretical_loss": 3.593230392363456, + "tokens_seen": 1179254784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003246038114343029, + "loss": 2.7311, + "theoretical_loss": 3.5932119638870645, + "tokens_seen": 1179320320 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245937813440321, + "loss": 2.4551, + "theoretical_loss": 3.5931935367214596, + "tokens_seen": 1179385856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032458375125376127, + "loss": 2.7065, + "theoretical_loss": 3.5931751108664756, + "tokens_seen": 1179451392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245737211634905, + "loss": 2.8443, + "theoretical_loss": 3.5931566863219464, + "tokens_seen": 1179516928 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032456369107321963, + "loss": 2.7742, + "theoretical_loss": 3.5931382630877065, + "tokens_seen": 1179582464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 654223, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4708504676818848, + "objective/train/theoretical_loss": 3.593119841163589, + "objective/train/tokens_used": 1200108000, + "theoretical_loss": 3.593119841163589, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032455366098294887, + "loss": 2.6758, + "theoretical_loss": 3.593119841163589, + "tokens_seen": 1179648000 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324543630892678, + "loss": 3.057, + "theoretical_loss": 3.593101420549429, + "tokens_seen": 1179713536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032453360080240723, + "loss": 2.6789, + "theoretical_loss": 3.593083001245059, + "tokens_seen": 1179779072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245235707121364, + "loss": 2.8285, + "theoretical_loss": 3.5930645832503147, + "tokens_seen": 1179844608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245135406218656, + "loss": 2.6795, + "theoretical_loss": 3.5930461665650295, + "tokens_seen": 1179910144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003245035105315948, + "loss": 2.6497, + "theoretical_loss": 3.5930277511890383, + "tokens_seen": 1179975680 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324493480441324, + "loss": 2.676, + "theoretical_loss": 3.593009337122174, + "tokens_seen": 1180041216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032448345035105314, + "loss": 2.731, + "theoretical_loss": 3.5929909243642726, + "tokens_seen": 1180106752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244734202607824, + "loss": 2.6065, + "theoretical_loss": 3.592972512915167, + "tokens_seen": 1180172288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244633901705115, + "loss": 2.7483, + "theoretical_loss": 3.592954102774691, + "tokens_seen": 1180237824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032445336008024074, + "loss": 2.4523, + "theoretical_loss": 3.5929356939426804, + "tokens_seen": 1180303360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244433299899699, + "loss": 2.4251, + "theoretical_loss": 3.592917286418969, + "tokens_seen": 1180368896 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244332998996991, + "loss": 2.6842, + "theoretical_loss": 3.592898880203391, + "tokens_seen": 1180434432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003244232698094283, + "loss": 2.6927, + "theoretical_loss": 3.592880475295781, + "tokens_seen": 1180499968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032441323971915746, + "loss": 2.7892, + "theoretical_loss": 3.5928620716959734, + "tokens_seen": 1180565504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032440320962888664, + "loss": 2.6003, + "theoretical_loss": 3.592843669403803, + "tokens_seen": 1180631040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243931795386159, + "loss": 2.5782, + "theoretical_loss": 3.5928252684191033, + "tokens_seen": 1180696576 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324383149448345, + "loss": 2.6193, + "theoretical_loss": 3.59280686874171, + "tokens_seen": 1180762112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032437311935807424, + "loss": 2.6897, + "theoretical_loss": 3.592788470371457, + "tokens_seen": 1180827648 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032436308926780337, + "loss": 2.7288, + "theoretical_loss": 3.5927700733081798, + "tokens_seen": 1180893184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243530591775326, + "loss": 2.6043, + "theoretical_loss": 3.5927516775517114, + "tokens_seen": 1180958720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243430290872618, + "loss": 2.7239, + "theoretical_loss": 3.5927332831018877, + "tokens_seen": 1181024256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032433299899699096, + "loss": 2.5689, + "theoretical_loss": 3.5927148899585433, + "tokens_seen": 1181089792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243229689067202, + "loss": 2.8611, + "theoretical_loss": 3.5926964981215126, + "tokens_seen": 1181155328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003243129388164494, + "loss": 2.531, + "theoretical_loss": 3.5926781075906304, + "tokens_seen": 1181220864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 655414, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6210334300994873, + "objective/train/theoretical_loss": 3.5926597183657316, + "objective/train/tokens_used": 1201746400, + "theoretical_loss": 3.5926597183657316, + "tokens_seen": 1181286400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032430290872617856, + "loss": 2.7083, + "theoretical_loss": 3.5926597183657316, + "tokens_seen": 1181286400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032429287863590774, + "loss": 2.6754, + "theoretical_loss": 3.592641330446651, + "tokens_seen": 1181351936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242828485456369, + "loss": 2.4938, + "theoretical_loss": 3.592622943833223, + "tokens_seen": 1181417472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242728184553661, + "loss": 2.5009, + "theoretical_loss": 3.5926045585252835, + "tokens_seen": 1181483008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032426278836509534, + "loss": 2.7818, + "theoretical_loss": 3.5925861745226664, + "tokens_seen": 1181548544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032425275827482447, + "loss": 2.7641, + "theoretical_loss": 3.592567791825207, + "tokens_seen": 1181614080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242427281845537, + "loss": 2.7128, + "theoretical_loss": 3.5925494104327402, + "tokens_seen": 1181679616 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032423269809428283, + "loss": 2.8377, + "theoretical_loss": 3.5925310303451012, + "tokens_seen": 1181745152 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032422266800401206, + "loss": 2.6549, + "theoretical_loss": 3.592512651562125, + "tokens_seen": 1181810688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032421263791374125, + "loss": 2.502, + "theoretical_loss": 3.592494274083646, + "tokens_seen": 1181876224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003242026078234704, + "loss": 2.6759, + "theoretical_loss": 3.5924758979095004, + "tokens_seen": 1181941760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241925777331996, + "loss": 2.6429, + "theoretical_loss": 3.5924575230395224, + "tokens_seen": 1182007296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032418254764292884, + "loss": 2.3383, + "theoretical_loss": 3.592439149473547, + "tokens_seen": 1182072832 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032417251755265797, + "loss": 2.6148, + "theoretical_loss": 3.5924207772114105, + "tokens_seen": 1182138368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241624874623872, + "loss": 2.6313, + "theoretical_loss": 3.5924024062529476, + "tokens_seen": 1182203904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032415245737211633, + "loss": 2.7198, + "theoretical_loss": 3.5923840365979927, + "tokens_seen": 1182269440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032414242728184557, + "loss": 2.6309, + "theoretical_loss": 3.592365668246382, + "tokens_seen": 1182334976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032413239719157475, + "loss": 2.937, + "theoretical_loss": 3.5923473011979508, + "tokens_seen": 1182400512 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032412236710130393, + "loss": 2.6589, + "theoretical_loss": 3.5923289354525334, + "tokens_seen": 1182466048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241123370110331, + "loss": 2.8447, + "theoretical_loss": 3.5923105710099663, + "tokens_seen": 1182531584 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003241023069207623, + "loss": 2.7621, + "theoretical_loss": 3.5922922078700847, + "tokens_seen": 1182597120 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032409227683049147, + "loss": 2.7971, + "theoretical_loss": 3.5922738460327235, + "tokens_seen": 1182662656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240822467402207, + "loss": 2.8789, + "theoretical_loss": 3.5922554854977182, + "tokens_seen": 1182728192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032407221664994984, + "loss": 2.6273, + "theoretical_loss": 3.592237126264905, + "tokens_seen": 1182793728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032406218655967907, + "loss": 2.8434, + "theoretical_loss": 3.5922187683341185, + "tokens_seen": 1182859264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 655986, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3905091285705566, + "objective/train/theoretical_loss": 3.5922004117051944, + "objective/train/tokens_used": 1203384800, + "theoretical_loss": 3.5922004117051944, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240521564694082, + "loss": 2.5661, + "theoretical_loss": 3.5922004117051944, + "tokens_seen": 1182924800 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032404212637913743, + "loss": 2.5477, + "theoretical_loss": 3.592182056377969, + "tokens_seen": 1182990336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240320962888666, + "loss": 2.7005, + "theoretical_loss": 3.592163702352277, + "tokens_seen": 1183055872 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240220661985958, + "loss": 2.7133, + "theoretical_loss": 3.5921453496279545, + "tokens_seen": 1183121408 + }, + { + "epoch": 0.36, + "learning_rate": 0.000324012036108325, + "loss": 2.7503, + "theoretical_loss": 3.5921269982048374, + "tokens_seen": 1183186944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003240020060180542, + "loss": 2.6778, + "theoretical_loss": 3.5921086480827604, + "tokens_seen": 1183252480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032399197592778334, + "loss": 2.7109, + "theoretical_loss": 3.59209029926156, + "tokens_seen": 1183318016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239819458375126, + "loss": 2.545, + "theoretical_loss": 3.592071951741072, + "tokens_seen": 1183383552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239719157472417, + "loss": 2.7016, + "theoretical_loss": 3.592053605521132, + "tokens_seen": 1183449088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032396188565697094, + "loss": 2.8656, + "theoretical_loss": 3.592035260601575, + "tokens_seen": 1183514624 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239518555667001, + "loss": 2.4547, + "theoretical_loss": 3.5920169169822382, + "tokens_seen": 1183580160 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239418254764293, + "loss": 2.7549, + "theoretical_loss": 3.5919985746629566, + "tokens_seen": 1183645696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239317953861585, + "loss": 2.8236, + "theoretical_loss": 3.5919802336435662, + "tokens_seen": 1183711232 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032392176529588766, + "loss": 2.4944, + "theoretical_loss": 3.5919618939239033, + "tokens_seen": 1183776768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032391173520561684, + "loss": 2.7984, + "theoretical_loss": 3.5919435555038035, + "tokens_seen": 1183842304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003239017051153461, + "loss": 2.6072, + "theoretical_loss": 3.5919252183831025, + "tokens_seen": 1183907840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238916750250752, + "loss": 2.6405, + "theoretical_loss": 3.591906882561637, + "tokens_seen": 1183973376 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032388164493480444, + "loss": 2.9206, + "theoretical_loss": 3.5918885480392424, + "tokens_seen": 1184038912 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032387161484453357, + "loss": 2.7311, + "theoretical_loss": 3.5918702148157555, + "tokens_seen": 1184104448 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238615847542628, + "loss": 2.7311, + "theoretical_loss": 3.591851882891012, + "tokens_seen": 1184169984 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323851554663992, + "loss": 2.7366, + "theoretical_loss": 3.5918335522648475, + "tokens_seen": 1184235520 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032384152457372116, + "loss": 2.3489, + "theoretical_loss": 3.5918152229370985, + "tokens_seen": 1184301056 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032383149448345034, + "loss": 2.6593, + "theoretical_loss": 3.591796894907602, + "tokens_seen": 1184366592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238214643931796, + "loss": 2.8426, + "theoretical_loss": 3.591778568176193, + "tokens_seen": 1184432128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003238114343029087, + "loss": 2.7458, + "theoretical_loss": 3.5917602427427084, + "tokens_seen": 1184497664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 656832, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.695746421813965, + "objective/train/theoretical_loss": 3.591741918606984, + "objective/train/tokens_used": 1205023200, + "theoretical_loss": 3.591741918606984, + "tokens_seen": 1184563200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032380140421263794, + "loss": 2.874, + "theoretical_loss": 3.591741918606984, + "tokens_seen": 1184563200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032379137412236707, + "loss": 2.6529, + "theoretical_loss": 3.5917235957688574, + "tokens_seen": 1184628736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237813440320963, + "loss": 2.6853, + "theoretical_loss": 3.591705274228163, + "tokens_seen": 1184694272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237713139418255, + "loss": 2.8111, + "theoretical_loss": 3.591686953984739, + "tokens_seen": 1184759808 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032376128385155467, + "loss": 2.6361, + "theoretical_loss": 3.5916686350384204, + "tokens_seen": 1184825344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032375125376128385, + "loss": 2.5834, + "theoretical_loss": 3.5916503173890444, + "tokens_seen": 1184890880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032374122367101303, + "loss": 2.905, + "theoretical_loss": 3.591632001036447, + "tokens_seen": 1184956416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237311935807422, + "loss": 2.6953, + "theoretical_loss": 3.591613685980465, + "tokens_seen": 1185021952 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032372116349047145, + "loss": 2.8276, + "theoretical_loss": 3.5915953722209353, + "tokens_seen": 1185087488 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032371113340020057, + "loss": 2.5766, + "theoretical_loss": 3.5915770597576935, + "tokens_seen": 1185153024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003237011033099298, + "loss": 2.586, + "theoretical_loss": 3.591558748590577, + "tokens_seen": 1185218560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032369107321965893, + "loss": 2.7271, + "theoretical_loss": 3.591540438719422, + "tokens_seen": 1185284096 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032368104312938817, + "loss": 2.8175, + "theoretical_loss": 3.5915221301440647, + "tokens_seen": 1185349632 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032367101303911735, + "loss": 2.823, + "theoretical_loss": 3.5915038228643423, + "tokens_seen": 1185415168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032366098294884653, + "loss": 2.7177, + "theoretical_loss": 3.5914855168800917, + "tokens_seen": 1185480704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236509528585757, + "loss": 2.648, + "theoretical_loss": 3.5914672121911493, + "tokens_seen": 1185546240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032364092276830495, + "loss": 2.8443, + "theoretical_loss": 3.591448908797352, + "tokens_seen": 1185611776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236308926780341, + "loss": 2.7258, + "theoretical_loss": 3.591430606698536, + "tokens_seen": 1185677312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236208625877633, + "loss": 2.5077, + "theoretical_loss": 3.591412305894539, + "tokens_seen": 1185742848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032361083249749244, + "loss": 2.7522, + "theoretical_loss": 3.5913940063851975, + "tokens_seen": 1185808384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003236008024072217, + "loss": 2.5417, + "theoretical_loss": 3.591375708170348, + "tokens_seen": 1185873920 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032359077231695085, + "loss": 2.8327, + "theoretical_loss": 3.5913574112498274, + "tokens_seen": 1185939456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032358074222668004, + "loss": 2.6164, + "theoretical_loss": 3.591339115623474, + "tokens_seen": 1186004992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032357071213640927, + "loss": 2.4979, + "theoretical_loss": 3.5913208212911227, + "tokens_seen": 1186070528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235606820461384, + "loss": 2.7355, + "theoretical_loss": 3.591302528252612, + "tokens_seen": 1186136064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 657301, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4484574794769287, + "objective/train/theoretical_loss": 3.5912842365077777, + "objective/train/tokens_used": 1206661600, + "theoretical_loss": 3.5912842365077777, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032355065195586763, + "loss": 2.6418, + "theoretical_loss": 3.5912842365077777, + "tokens_seen": 1186201600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235406218655968, + "loss": 2.7045, + "theoretical_loss": 3.591265946056458, + "tokens_seen": 1186267136 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323530591775326, + "loss": 2.6399, + "theoretical_loss": 3.5912476568984895, + "tokens_seen": 1186332672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235205616850552, + "loss": 2.7264, + "theoretical_loss": 3.5912293690337096, + "tokens_seen": 1186398208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003235105315947844, + "loss": 2.6608, + "theoretical_loss": 3.5912110824619545, + "tokens_seen": 1186463744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032350050150451354, + "loss": 2.6546, + "theoretical_loss": 3.591192797183062, + "tokens_seen": 1186529280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234904714142428, + "loss": 2.6396, + "theoretical_loss": 3.59117451319687, + "tokens_seen": 1186594816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234804413239719, + "loss": 2.711, + "theoretical_loss": 3.5911562305032145, + "tokens_seen": 1186660352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032347041123370114, + "loss": 2.735, + "theoretical_loss": 3.5911379491019333, + "tokens_seen": 1186725888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234603811434303, + "loss": 2.7617, + "theoretical_loss": 3.5911196689928637, + "tokens_seen": 1186791424 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234503510531595, + "loss": 2.6371, + "theoretical_loss": 3.591101390175843, + "tokens_seen": 1186856960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234403209628887, + "loss": 2.6863, + "theoretical_loss": 3.5910831126507086, + "tokens_seen": 1186922496 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032343029087261786, + "loss": 2.7775, + "theoretical_loss": 3.5910648364172975, + "tokens_seen": 1186988032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032342026078234704, + "loss": 2.9409, + "theoretical_loss": 3.591046561475447, + "tokens_seen": 1187053568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234102306920763, + "loss": 2.9129, + "theoretical_loss": 3.5910282878249955, + "tokens_seen": 1187119104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003234002006018054, + "loss": 2.7658, + "theoretical_loss": 3.5910100154657796, + "tokens_seen": 1187184640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032339017051153464, + "loss": 2.5633, + "theoretical_loss": 3.590991744397637, + "tokens_seen": 1187250176 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032338014042126377, + "loss": 2.6758, + "theoretical_loss": 3.5909734746204056, + "tokens_seen": 1187315712 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323370110330993, + "loss": 2.6059, + "theoretical_loss": 3.5909552061339225, + "tokens_seen": 1187381248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233600802407222, + "loss": 2.6783, + "theoretical_loss": 3.5909369389380252, + "tokens_seen": 1187446784 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032335005015045136, + "loss": 2.5929, + "theoretical_loss": 3.5909186730325517, + "tokens_seen": 1187512320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032334002006018054, + "loss": 2.6622, + "theoretical_loss": 3.590900408417339, + "tokens_seen": 1187577856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233299899699098, + "loss": 2.4147, + "theoretical_loss": 3.5908821450922255, + "tokens_seen": 1187643392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003233199598796389, + "loss": 2.5664, + "theoretical_loss": 3.5908638830570485, + "tokens_seen": 1187708928 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032330992978936814, + "loss": 2.6783, + "theoretical_loss": 3.5908456223116456, + "tokens_seen": 1187774464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 658636, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8977458477020264, + "objective/train/theoretical_loss": 3.590827362855855, + "objective/train/tokens_used": 1208300000, + "theoretical_loss": 3.590827362855855, + "tokens_seen": 1187840000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032329989969909727, + "loss": 2.8993, + "theoretical_loss": 3.590827362855855, + "tokens_seen": 1187840000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232898696088265, + "loss": 2.895, + "theoretical_loss": 3.590809104689514, + "tokens_seen": 1187905536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232798395185557, + "loss": 2.6605, + "theoretical_loss": 3.5907908478124613, + "tokens_seen": 1187971072 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032326980942828487, + "loss": 2.7268, + "theoretical_loss": 3.5907725922245337, + "tokens_seen": 1188036608 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032325977933801405, + "loss": 2.4671, + "theoretical_loss": 3.5907543379255693, + "tokens_seen": 1188102144 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032324974924774323, + "loss": 2.5579, + "theoretical_loss": 3.590736084915406, + "tokens_seen": 1188167680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003232397191574724, + "loss": 2.5451, + "theoretical_loss": 3.5907178331938825, + "tokens_seen": 1188233216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032322968906720165, + "loss": 2.8114, + "theoretical_loss": 3.5906995827608355, + "tokens_seen": 1188298752 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032321965897693077, + "loss": 2.7601, + "theoretical_loss": 3.5906813336161036, + "tokens_seen": 1188364288 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032320962888666, + "loss": 2.8106, + "theoretical_loss": 3.590663085759525, + "tokens_seen": 1188429824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032319959879638913, + "loss": 2.6096, + "theoretical_loss": 3.5906448391909382, + "tokens_seen": 1188495360 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032318956870611837, + "loss": 2.7287, + "theoretical_loss": 3.59062659391018, + "tokens_seen": 1188560896 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032317953861584755, + "loss": 2.734, + "theoretical_loss": 3.5906083499170895, + "tokens_seen": 1188626432 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032316950852557673, + "loss": 2.9007, + "theoretical_loss": 3.590590107211504, + "tokens_seen": 1188691968 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231594784353059, + "loss": 2.624, + "theoretical_loss": 3.5905718657932626, + "tokens_seen": 1188757504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032314944834503515, + "loss": 2.5711, + "theoretical_loss": 3.590553625662203, + "tokens_seen": 1188823040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231394182547643, + "loss": 2.6533, + "theoretical_loss": 3.5905353868181633, + "tokens_seen": 1188888576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231293881644935, + "loss": 2.6393, + "theoretical_loss": 3.5905171492609815, + "tokens_seen": 1188954112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032311935807422264, + "loss": 2.7133, + "theoretical_loss": 3.5904989129904967, + "tokens_seen": 1189019648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003231093279839519, + "loss": 2.5837, + "theoretical_loss": 3.5904806780065472, + "tokens_seen": 1189085184 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032309929789368105, + "loss": 2.8506, + "theoretical_loss": 3.5904624443089705, + "tokens_seen": 1189150720 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032308926780341024, + "loss": 2.9039, + "theoretical_loss": 3.590444211897606, + "tokens_seen": 1189216256 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230792377131394, + "loss": 2.7202, + "theoretical_loss": 3.5904259807722907, + "tokens_seen": 1189281792 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230692076228686, + "loss": 2.6139, + "theoretical_loss": 3.5904077509328642, + "tokens_seen": 1189347328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230591775325978, + "loss": 2.5795, + "theoretical_loss": 3.590389522379165, + "tokens_seen": 1189412864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 659771, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9299919605255127, + "objective/train/theoretical_loss": 3.5903712951110305, + "objective/train/tokens_used": 1209938400, + "theoretical_loss": 3.5903712951110305, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.000323049147442327, + "loss": 2.7602, + "theoretical_loss": 3.5903712951110305, + "tokens_seen": 1189478400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032303911735205614, + "loss": 2.4382, + "theoretical_loss": 3.5903530691283, + "tokens_seen": 1189543936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230290872617854, + "loss": 2.8542, + "theoretical_loss": 3.590334844430812, + "tokens_seen": 1189609472 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003230190571715145, + "loss": 2.6869, + "theoretical_loss": 3.590316621018405, + "tokens_seen": 1189675008 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032300902708124374, + "loss": 2.7318, + "theoretical_loss": 3.5902983988909183, + "tokens_seen": 1189740544 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229989969909729, + "loss": 2.6897, + "theoretical_loss": 3.5902801780481886, + "tokens_seen": 1189806080 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229889669007021, + "loss": 2.5943, + "theoretical_loss": 3.590261958490057, + "tokens_seen": 1189871616 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229789368104313, + "loss": 2.5884, + "theoretical_loss": 3.5902437402163603, + "tokens_seen": 1189937152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229689067201605, + "loss": 2.7583, + "theoretical_loss": 3.5902255232269376, + "tokens_seen": 1190002688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032295887662988964, + "loss": 2.6598, + "theoretical_loss": 3.5902073075216285, + "tokens_seen": 1190068224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229488465396189, + "loss": 2.6565, + "theoretical_loss": 3.590189093100271, + "tokens_seen": 1190133760 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322938816449348, + "loss": 2.7323, + "theoretical_loss": 3.590170879962704, + "tokens_seen": 1190199296 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032292878635907724, + "loss": 2.5401, + "theoretical_loss": 3.590152668108767, + "tokens_seen": 1190264832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229187562688064, + "loss": 2.7671, + "theoretical_loss": 3.590134457538298, + "tokens_seen": 1190330368 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003229087261785356, + "loss": 2.6562, + "theoretical_loss": 3.590116248251136, + "tokens_seen": 1190395904 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228986960882648, + "loss": 2.5468, + "theoretical_loss": 3.59009804024712, + "tokens_seen": 1190461440 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032288866599799397, + "loss": 2.8533, + "theoretical_loss": 3.5900798335260897, + "tokens_seen": 1190526976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032287863590772315, + "loss": 2.8279, + "theoretical_loss": 3.590061628087883, + "tokens_seen": 1190592512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228686058174524, + "loss": 2.6782, + "theoretical_loss": 3.5900434239323396, + "tokens_seen": 1190658048 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228585757271815, + "loss": 2.668, + "theoretical_loss": 3.590025221059298, + "tokens_seen": 1190723584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032284854563691074, + "loss": 2.6663, + "theoretical_loss": 3.5900070194685982, + "tokens_seen": 1190789120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228385155466399, + "loss": 2.9798, + "theoretical_loss": 3.589988819160078, + "tokens_seen": 1190854656 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003228284854563691, + "loss": 2.5456, + "theoretical_loss": 3.5899706201335775, + "tokens_seen": 1190920192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032281845536609834, + "loss": 2.7142, + "theoretical_loss": 3.5899524223889356, + "tokens_seen": 1190985728 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032280842527582747, + "loss": 2.6882, + "theoretical_loss": 3.589934225925991, + "tokens_seen": 1191051264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 660497, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.826659917831421, + "objective/train/theoretical_loss": 3.589916030744584, + "objective/train/tokens_used": 1211576800, + "theoretical_loss": 3.589916030744584, + "tokens_seen": 1191116800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227983951855567, + "loss": 2.8398, + "theoretical_loss": 3.589916030744584, + "tokens_seen": 1191116800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227883650952859, + "loss": 2.536, + "theoretical_loss": 3.589897836844553, + "tokens_seen": 1191182336 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032277833500501507, + "loss": 2.5995, + "theoretical_loss": 3.589879644225737, + "tokens_seen": 1191247872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032276830491474425, + "loss": 2.7609, + "theoretical_loss": 3.589861452887976, + "tokens_seen": 1191313408 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032275827482447343, + "loss": 2.5637, + "theoretical_loss": 3.5898432628311094, + "tokens_seen": 1191378944 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227482447342026, + "loss": 2.8383, + "theoretical_loss": 3.5898250740549758, + "tokens_seen": 1191444480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032273821464393185, + "loss": 2.5828, + "theoretical_loss": 3.5898068865594155, + "tokens_seen": 1191510016 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032272818455366097, + "loss": 2.7367, + "theoretical_loss": 3.589788700344267, + "tokens_seen": 1191575552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003227181544633902, + "loss": 2.7134, + "theoretical_loss": 3.5897705154093704, + "tokens_seen": 1191641088 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032270812437311933, + "loss": 2.535, + "theoretical_loss": 3.589752331754565, + "tokens_seen": 1191706624 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032269809428284857, + "loss": 2.6662, + "theoretical_loss": 3.58973414937969, + "tokens_seen": 1191772160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032268806419257775, + "loss": 2.5705, + "theoretical_loss": 3.589715968284585, + "tokens_seen": 1191837696 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032267803410230693, + "loss": 2.7252, + "theoretical_loss": 3.58969778846909, + "tokens_seen": 1191903232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226680040120361, + "loss": 2.596, + "theoretical_loss": 3.589679609933045, + "tokens_seen": 1191968768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032265797392176535, + "loss": 2.5493, + "theoretical_loss": 3.589661432676288, + "tokens_seen": 1192034304 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226479438314945, + "loss": 2.5605, + "theoretical_loss": 3.58964325669866, + "tokens_seen": 1192099840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226379137412237, + "loss": 2.7074, + "theoretical_loss": 3.589625082, + "tokens_seen": 1192165376 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032262788365095284, + "loss": 2.7322, + "theoretical_loss": 3.5896069085801483, + "tokens_seen": 1192230912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003226178535606821, + "loss": 2.8322, + "theoretical_loss": 3.5895887364389445, + "tokens_seen": 1192296448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032260782347041125, + "loss": 2.6919, + "theoretical_loss": 3.589570565576228, + "tokens_seen": 1192361984 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032259779338014044, + "loss": 2.708, + "theoretical_loss": 3.5895523959918383, + "tokens_seen": 1192427520 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225877632898696, + "loss": 2.7512, + "theoretical_loss": 3.5895342276856157, + "tokens_seen": 1192493056 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225777331995988, + "loss": 2.5763, + "theoretical_loss": 3.5895160606574006, + "tokens_seen": 1192558592 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322567703109328, + "loss": 2.7169, + "theoretical_loss": 3.5894978949070317, + "tokens_seen": 1192624128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225576730190572, + "loss": 2.4533, + "theoretical_loss": 3.58947973043435, + "tokens_seen": 1192689664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 661795, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.610649347305298, + "objective/train/theoretical_loss": 3.5894615672391947, + "objective/train/tokens_used": 1213215200, + "theoretical_loss": 3.5894615672391947, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032254764292878634, + "loss": 2.6943, + "theoretical_loss": 3.5894615672391947, + "tokens_seen": 1192755200 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225376128385156, + "loss": 2.7775, + "theoretical_loss": 3.589443405321406, + "tokens_seen": 1192820736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225275827482447, + "loss": 2.541, + "theoretical_loss": 3.5894252446808244, + "tokens_seen": 1192886272 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032251755265797394, + "loss": 2.6342, + "theoretical_loss": 3.589407085317289, + "tokens_seen": 1192951808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003225075225677031, + "loss": 2.2844, + "theoretical_loss": 3.5893889272306403, + "tokens_seen": 1193017344 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224974924774323, + "loss": 2.7089, + "theoretical_loss": 3.589370770420718, + "tokens_seen": 1193082880 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224874623871615, + "loss": 2.4681, + "theoretical_loss": 3.589352614887363, + "tokens_seen": 1193148416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224774322968907, + "loss": 2.833, + "theoretical_loss": 3.5893344606304147, + "tokens_seen": 1193213952 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032246740220661984, + "loss": 2.5555, + "theoretical_loss": 3.589316307649714, + "tokens_seen": 1193279488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224573721163491, + "loss": 2.656, + "theoretical_loss": 3.5892981559451003, + "tokens_seen": 1193345024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224473420260782, + "loss": 2.5065, + "theoretical_loss": 3.5892800055164145, + "tokens_seen": 1193410560 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032243731193580744, + "loss": 2.4987, + "theoretical_loss": 3.589261856363496, + "tokens_seen": 1193476096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224272818455366, + "loss": 2.7401, + "theoretical_loss": 3.589243708486186, + "tokens_seen": 1193541632 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003224172517552658, + "loss": 2.5925, + "theoretical_loss": 3.5892255618843247, + "tokens_seen": 1193607168 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322407221664995, + "loss": 2.8549, + "theoretical_loss": 3.589207416557752, + "tokens_seen": 1193672704 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032239719157472417, + "loss": 2.7572, + "theoretical_loss": 3.589189272506308, + "tokens_seen": 1193738240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032238716148445335, + "loss": 2.676, + "theoretical_loss": 3.5891711297298334, + "tokens_seen": 1193803776 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223771313941826, + "loss": 2.5692, + "theoretical_loss": 3.5891529882281694, + "tokens_seen": 1193869312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223671013039117, + "loss": 2.6654, + "theoretical_loss": 3.5891348480011556, + "tokens_seen": 1193934848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032235707121364095, + "loss": 2.4853, + "theoretical_loss": 3.589116709048633, + "tokens_seen": 1194000384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223470411233701, + "loss": 2.5927, + "theoretical_loss": 3.589098571370441, + "tokens_seen": 1194065920 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223370110330993, + "loss": 2.6482, + "theoretical_loss": 3.589080434966421, + "tokens_seen": 1194131456 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003223269809428285, + "loss": 2.5284, + "theoretical_loss": 3.5890622998364146, + "tokens_seen": 1194196992 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032231695085255767, + "loss": 2.65, + "theoretical_loss": 3.58904416598026, + "tokens_seen": 1194262528 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032230692076228685, + "loss": 2.4034, + "theoretical_loss": 3.5890260333977997, + "tokens_seen": 1194328064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 662071, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3905045986175537, + "objective/train/theoretical_loss": 3.589007902088874, + "objective/train/tokens_used": 1214853600, + "theoretical_loss": 3.589007902088874, + "tokens_seen": 1194393600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222968906720161, + "loss": 2.4546, + "theoretical_loss": 3.589007902088874, + "tokens_seen": 1194393600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222868605817452, + "loss": 2.8831, + "theoretical_loss": 3.5889897720533233, + "tokens_seen": 1194459136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032227683049147445, + "loss": 2.6318, + "theoretical_loss": 3.5889716432909875, + "tokens_seen": 1194524672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222668004012036, + "loss": 2.4259, + "theoretical_loss": 3.588953515801709, + "tokens_seen": 1194590208 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222567703109328, + "loss": 2.7315, + "theoretical_loss": 3.5889353895853278, + "tokens_seen": 1194655744 + }, + { + "epoch": 0.36, + "learning_rate": 0.000322246740220662, + "loss": 2.7919, + "theoretical_loss": 3.588917264641685, + "tokens_seen": 1194721280 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032223671013039117, + "loss": 2.5828, + "theoretical_loss": 3.5888991409706206, + "tokens_seen": 1194786816 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032222668004012035, + "loss": 2.6261, + "theoretical_loss": 3.5888810185719757, + "tokens_seen": 1194852352 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032221664994984953, + "loss": 2.4561, + "theoretical_loss": 3.5888628974455914, + "tokens_seen": 1194917888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003222066198595787, + "loss": 2.5928, + "theoretical_loss": 3.5888447775913095, + "tokens_seen": 1194983424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032219658976930795, + "loss": 2.5253, + "theoretical_loss": 3.5888266590089692, + "tokens_seen": 1195048960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221865596790371, + "loss": 2.6944, + "theoretical_loss": 3.588808541698413, + "tokens_seen": 1195114496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221765295887663, + "loss": 2.4546, + "theoretical_loss": 3.588790425659481, + "tokens_seen": 1195180032 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221664994984955, + "loss": 2.6356, + "theoretical_loss": 3.588772310892015, + "tokens_seen": 1195245568 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221564694082247, + "loss": 2.6211, + "theoretical_loss": 3.5887541973958546, + "tokens_seen": 1195311104 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032214643931795386, + "loss": 2.758, + "theoretical_loss": 3.5887360851708423, + "tokens_seen": 1195376640 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032213640922768304, + "loss": 2.7939, + "theoretical_loss": 3.5887179742168187, + "tokens_seen": 1195442176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221263791374122, + "loss": 2.616, + "theoretical_loss": 3.588699864533625, + "tokens_seen": 1195507712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032211634904714145, + "loss": 2.6652, + "theoretical_loss": 3.5886817561211024, + "tokens_seen": 1195573248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003221063189568706, + "loss": 2.7012, + "theoretical_loss": 3.588663648979092, + "tokens_seen": 1195638784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220962888665998, + "loss": 2.7739, + "theoretical_loss": 3.588645543107435, + "tokens_seen": 1195704320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032208625877632894, + "loss": 2.8342, + "theoretical_loss": 3.5886274385059727, + "tokens_seen": 1195769856 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220762286860582, + "loss": 2.6931, + "theoretical_loss": 3.5886093351745467, + "tokens_seen": 1195835392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220661985957874, + "loss": 2.5972, + "theoretical_loss": 3.5885912331129974, + "tokens_seen": 1195900928 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032205616850551654, + "loss": 2.6297, + "theoretical_loss": 3.5885731323211676, + "tokens_seen": 1195966464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 663277, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6097090244293213, + "objective/train/theoretical_loss": 3.5885550327988973, + "objective/train/tokens_used": 1216492000, + "theoretical_loss": 3.5885550327988973, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220461384152458, + "loss": 2.7212, + "theoretical_loss": 3.5885550327988973, + "tokens_seen": 1196032000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220361083249749, + "loss": 2.7139, + "theoretical_loss": 3.5885369345460285, + "tokens_seen": 1196097536 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032202607823470414, + "loss": 2.5494, + "theoretical_loss": 3.5885188375624026, + "tokens_seen": 1196163072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220160481444333, + "loss": 2.6198, + "theoretical_loss": 3.588500741847861, + "tokens_seen": 1196228608 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003220060180541625, + "loss": 2.9059, + "theoretical_loss": 3.5884826474022455, + "tokens_seen": 1196294144 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219959879638917, + "loss": 2.7009, + "theoretical_loss": 3.588464554225397, + "tokens_seen": 1196359680 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219859578736209, + "loss": 2.7996, + "theoretical_loss": 3.5884464623171572, + "tokens_seen": 1196425216 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032197592778335004, + "loss": 2.7252, + "theoretical_loss": 3.588428371677368, + "tokens_seen": 1196490752 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219658976930793, + "loss": 2.623, + "theoretical_loss": 3.588410282305871, + "tokens_seen": 1196556288 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219558676028084, + "loss": 2.7392, + "theoretical_loss": 3.588392194202507, + "tokens_seen": 1196621824 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032194583751253764, + "loss": 2.8584, + "theoretical_loss": 3.5883741073671187, + "tokens_seen": 1196687360 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219358074222668, + "loss": 2.6099, + "theoretical_loss": 3.5883560217995476, + "tokens_seen": 1196752896 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321925777331996, + "loss": 2.8514, + "theoretical_loss": 3.588337937499635, + "tokens_seen": 1196818432 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003219157472417252, + "loss": 2.5845, + "theoretical_loss": 3.5883198544672226, + "tokens_seen": 1196883968 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032190571715145437, + "loss": 2.6211, + "theoretical_loss": 3.588301772702153, + "tokens_seen": 1196949504 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032189568706118355, + "loss": 2.783, + "theoretical_loss": 3.588283692204267, + "tokens_seen": 1197015040 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218856569709128, + "loss": 2.7052, + "theoretical_loss": 3.5882656129734065, + "tokens_seen": 1197080576 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218756268806419, + "loss": 2.7137, + "theoretical_loss": 3.5882475350094136, + "tokens_seen": 1197146112 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032186559679037115, + "loss": 2.5264, + "theoretical_loss": 3.5882294583121306, + "tokens_seen": 1197211648 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218555667001003, + "loss": 2.5417, + "theoretical_loss": 3.5882113828813993, + "tokens_seen": 1197277184 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218455366098295, + "loss": 2.6667, + "theoretical_loss": 3.588193308717061, + "tokens_seen": 1197342720 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218355065195587, + "loss": 2.6857, + "theoretical_loss": 3.5881752358189583, + "tokens_seen": 1197408256 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032182547642928787, + "loss": 2.7712, + "theoretical_loss": 3.588157164186933, + "tokens_seen": 1197473792 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032181544633901705, + "loss": 2.6639, + "theoretical_loss": 3.5881390938208266, + "tokens_seen": 1197539328 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003218054162487463, + "loss": 2.7148, + "theoretical_loss": 3.588121024720482, + "tokens_seen": 1197604864 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 664536, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6411828994750977, + "objective/train/theoretical_loss": 3.588102956885741, + "objective/train/tokens_used": 1218130400, + "theoretical_loss": 3.588102956885741, + "tokens_seen": 1197670400 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217953861584754, + "loss": 2.83, + "theoretical_loss": 3.588102956885741, + "tokens_seen": 1197670400 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032178535606820465, + "loss": 2.5972, + "theoretical_loss": 3.5880848903164457, + "tokens_seen": 1197735936 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217753259779338, + "loss": 2.7223, + "theoretical_loss": 3.5880668250124375, + "tokens_seen": 1197801472 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321765295887663, + "loss": 2.7356, + "theoretical_loss": 3.5880487609735594, + "tokens_seen": 1197867008 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217552657973922, + "loss": 2.7346, + "theoretical_loss": 3.588030698199654, + "tokens_seen": 1197932544 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032174523570712137, + "loss": 2.575, + "theoretical_loss": 3.5880126366905625, + "tokens_seen": 1197998080 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032173520561685055, + "loss": 2.4085, + "theoretical_loss": 3.5879945764461274, + "tokens_seen": 1198063616 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032172517552657974, + "loss": 2.6027, + "theoretical_loss": 3.5879765174661915, + "tokens_seen": 1198129152 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003217151454363089, + "loss": 2.6716, + "theoretical_loss": 3.5879584597505967, + "tokens_seen": 1198194688 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032170511534603815, + "loss": 2.61, + "theoretical_loss": 3.5879404032991853, + "tokens_seen": 1198260224 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216950852557673, + "loss": 2.5906, + "theoretical_loss": 3.5879223481117997, + "tokens_seen": 1198325760 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216850551654965, + "loss": 2.5357, + "theoretical_loss": 3.5879042941882826, + "tokens_seen": 1198391296 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216750250752257, + "loss": 2.8647, + "theoretical_loss": 3.5878862415284765, + "tokens_seen": 1198456832 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216649949849549, + "loss": 2.5841, + "theoretical_loss": 3.5878681901322227, + "tokens_seen": 1198522368 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032165496489468406, + "loss": 2.6733, + "theoretical_loss": 3.5878501399993645, + "tokens_seen": 1198587904 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032164493480441324, + "loss": 2.5762, + "theoretical_loss": 3.587832091129745, + "tokens_seen": 1198653440 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216349047141424, + "loss": 2.6126, + "theoretical_loss": 3.587814043523206, + "tokens_seen": 1198718976 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032162487462387165, + "loss": 2.3205, + "theoretical_loss": 3.5877959971795903, + "tokens_seen": 1198784512 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003216148445336008, + "loss": 2.6872, + "theoretical_loss": 3.58777795209874, + "tokens_seen": 1198850048 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032160481444333, + "loss": 2.7524, + "theoretical_loss": 3.587759908280498, + "tokens_seen": 1198915584 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032159478435305914, + "loss": 2.7265, + "theoretical_loss": 3.5877418657247073, + "tokens_seen": 1198981120 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215847542627884, + "loss": 2.7187, + "theoretical_loss": 3.5877238244312104, + "tokens_seen": 1199046656 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032157472417251756, + "loss": 2.8152, + "theoretical_loss": 3.5877057843998497, + "tokens_seen": 1199112192 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032156469408224674, + "loss": 2.6784, + "theoretical_loss": 3.5876877456304683, + "tokens_seen": 1199177728 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215546639919759, + "loss": 2.6942, + "theoretical_loss": 3.5876697081229088, + "tokens_seen": 1199243264 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 665055, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5700125694274902, + "objective/train/theoretical_loss": 3.587651671877014, + "objective/train/tokens_used": 1219768800, + "theoretical_loss": 3.587651671877014, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215446339017051, + "loss": 2.676, + "theoretical_loss": 3.587651671877014, + "tokens_seen": 1199308800 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215346038114343, + "loss": 2.5606, + "theoretical_loss": 3.587633636892627, + "tokens_seen": 1199374336 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215245737211635, + "loss": 2.5459, + "theoretical_loss": 3.5876156031695903, + "tokens_seen": 1199439872 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032151454363089265, + "loss": 2.517, + "theoretical_loss": 3.587597570707746, + "tokens_seen": 1199505408 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003215045135406219, + "loss": 2.6024, + "theoretical_loss": 3.5875795395069385, + "tokens_seen": 1199570944 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032149448345035106, + "loss": 2.8159, + "theoretical_loss": 3.58756150956701, + "tokens_seen": 1199636480 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032148445336008024, + "loss": 2.6025, + "theoretical_loss": 3.5875434808878035, + "tokens_seen": 1199702016 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214744232698094, + "loss": 2.4649, + "theoretical_loss": 3.587525453469162, + "tokens_seen": 1199767552 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214643931795386, + "loss": 2.8485, + "theoretical_loss": 3.5875074273109284, + "tokens_seen": 1199833088 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214543630892678, + "loss": 2.6042, + "theoretical_loss": 3.587489402412946, + "tokens_seen": 1199898624 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321444332998997, + "loss": 2.7668, + "theoretical_loss": 3.5874713787750574, + "tokens_seen": 1199964160 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032143430290872615, + "loss": 2.9209, + "theoretical_loss": 3.587453356397106, + "tokens_seen": 1200029696 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214242728184554, + "loss": 2.8785, + "theoretical_loss": 3.587435335278935, + "tokens_seen": 1200095232 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003214142427281845, + "loss": 2.4521, + "theoretical_loss": 3.587417315420388, + "tokens_seen": 1200160768 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032140421263791375, + "loss": 2.8288, + "theoretical_loss": 3.5873992968213066, + "tokens_seen": 1200226304 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032139418254764293, + "loss": 2.6307, + "theoretical_loss": 3.5873812794815363, + "tokens_seen": 1200291840 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213841524573721, + "loss": 2.8476, + "theoretical_loss": 3.587363263400918, + "tokens_seen": 1200357376 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213741223671013, + "loss": 2.7485, + "theoretical_loss": 3.5873452485792967, + "tokens_seen": 1200422912 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213640922768305, + "loss": 2.7168, + "theoretical_loss": 3.587327235016515, + "tokens_seen": 1200488448 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032135406218655965, + "loss": 2.4365, + "theoretical_loss": 3.587309222712416, + "tokens_seen": 1200553984 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213440320962889, + "loss": 2.6422, + "theoretical_loss": 3.5872912116668436, + "tokens_seen": 1200619520 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321334002006018, + "loss": 2.6353, + "theoretical_loss": 3.5872732018796407, + "tokens_seen": 1200685056 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032132397191574725, + "loss": 2.6725, + "theoretical_loss": 3.587255193350651, + "tokens_seen": 1200750592 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213139418254765, + "loss": 2.6841, + "theoretical_loss": 3.587237186079718, + "tokens_seen": 1200816128 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003213039117352056, + "loss": 2.5766, + "theoretical_loss": 3.587219180066685, + "tokens_seen": 1200881664 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 666208, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7520852088928223, + "objective/train/theoretical_loss": 3.587201175311395, + "objective/train/tokens_used": 1221407200, + "theoretical_loss": 3.587201175311395, + "tokens_seen": 1200947200 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032129388164493485, + "loss": 2.8138, + "theoretical_loss": 3.587201175311395, + "tokens_seen": 1200947200 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321283851554664, + "loss": 2.8434, + "theoretical_loss": 3.5871831718136926, + "tokens_seen": 1201012736 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212738214643932, + "loss": 2.8096, + "theoretical_loss": 3.5871651695734204, + "tokens_seen": 1201078272 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212637913741224, + "loss": 2.446, + "theoretical_loss": 3.5871471685904224, + "tokens_seen": 1201143808 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212537612838516, + "loss": 2.709, + "theoretical_loss": 3.5871291688645424, + "tokens_seen": 1201209344 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032124373119358075, + "loss": 2.5374, + "theoretical_loss": 3.5871111703956235, + "tokens_seen": 1201274880 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032123370110330994, + "loss": 2.5882, + "theoretical_loss": 3.5870931731835096, + "tokens_seen": 1201340416 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212236710130391, + "loss": 2.496, + "theoretical_loss": 3.5870751772280443, + "tokens_seen": 1201405952 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032121364092276835, + "loss": 2.4976, + "theoretical_loss": 3.5870571825290716, + "tokens_seen": 1201471488 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003212036108324975, + "loss": 2.6233, + "theoretical_loss": 3.5870391890864353, + "tokens_seen": 1201537024 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211935807422267, + "loss": 2.4834, + "theoretical_loss": 3.587021196899978, + "tokens_seen": 1201602560 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211835506519559, + "loss": 2.5302, + "theoretical_loss": 3.5870032059695456, + "tokens_seen": 1201668096 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211735205616851, + "loss": 2.6988, + "theoretical_loss": 3.58698521629498, + "tokens_seen": 1201733632 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032116349047141426, + "loss": 2.5982, + "theoretical_loss": 3.586967227876126, + "tokens_seen": 1201799168 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032115346038114344, + "loss": 2.7329, + "theoretical_loss": 3.586949240712827, + "tokens_seen": 1201864704 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211434302908726, + "loss": 2.6829, + "theoretical_loss": 3.5869312548049277, + "tokens_seen": 1201930240 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032113340020060185, + "loss": 2.883, + "theoretical_loss": 3.5869132701522712, + "tokens_seen": 1201995776 + }, + { + "epoch": 0.36, + "learning_rate": 0.000321123370110331, + "loss": 2.6068, + "theoretical_loss": 3.586895286754702, + "tokens_seen": 1202061312 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003211133400200602, + "loss": 2.6619, + "theoretical_loss": 3.586877304612064, + "tokens_seen": 1202126848 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032110330992978934, + "loss": 2.6131, + "theoretical_loss": 3.5868593237242004, + "tokens_seen": 1202192384 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210932798395186, + "loss": 2.737, + "theoretical_loss": 3.5868413440909563, + "tokens_seen": 1202257920 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032108324974924776, + "loss": 2.747, + "theoretical_loss": 3.5868233657121755, + "tokens_seen": 1202323456 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032107321965897694, + "loss": 2.7392, + "theoretical_loss": 3.5868053885877016, + "tokens_seen": 1202388992 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210631895687061, + "loss": 2.5638, + "theoretical_loss": 3.5867874127173796, + "tokens_seen": 1202454528 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210531594784353, + "loss": 2.6974, + "theoretical_loss": 3.5867694381010535, + "tokens_seen": 1202520064 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 666794, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.74890398979187, + "objective/train/theoretical_loss": 3.5867514647385663, + "objective/train/tokens_used": 1223045600, + "theoretical_loss": 3.5867514647385663, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210431293881645, + "loss": 2.7497, + "theoretical_loss": 3.5867514647385663, + "tokens_seen": 1202585600 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210330992978937, + "loss": 2.5142, + "theoretical_loss": 3.586733492629764, + "tokens_seen": 1202651136 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032102306920762285, + "loss": 2.6491, + "theoretical_loss": 3.5867155217744893, + "tokens_seen": 1202716672 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003210130391173521, + "loss": 2.8934, + "theoretical_loss": 3.5866975521725877, + "tokens_seen": 1202782208 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032100300902708126, + "loss": 2.7662, + "theoretical_loss": 3.5866795838239023, + "tokens_seen": 1202847744 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032099297893681044, + "loss": 2.738, + "theoretical_loss": 3.5866616167282785, + "tokens_seen": 1202913280 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209829488465396, + "loss": 2.6258, + "theoretical_loss": 3.58664365088556, + "tokens_seen": 1202978816 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209729187562688, + "loss": 2.6219, + "theoretical_loss": 3.5866256862955916, + "tokens_seen": 1203044352 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320962888665998, + "loss": 2.5657, + "theoretical_loss": 3.5866077229582176, + "tokens_seen": 1203109888 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209528585757272, + "loss": 2.6135, + "theoretical_loss": 3.5865897608732817, + "tokens_seen": 1203175424 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032094282848545635, + "loss": 2.7004, + "theoretical_loss": 3.5865718000406295, + "tokens_seen": 1203240960 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209327983951856, + "loss": 2.7194, + "theoretical_loss": 3.586553840460105, + "tokens_seen": 1203306496 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003209227683049147, + "loss": 2.6563, + "theoretical_loss": 3.5865358821315527, + "tokens_seen": 1203372032 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032091273821464395, + "loss": 2.797, + "theoretical_loss": 3.5865179250548174, + "tokens_seen": 1203437568 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032090270812437313, + "loss": 2.6798, + "theoretical_loss": 3.586499969229743, + "tokens_seen": 1203503104 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208926780341023, + "loss": 2.8114, + "theoretical_loss": 3.5864820146561747, + "tokens_seen": 1203568640 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208826479438315, + "loss": 2.5543, + "theoretical_loss": 3.586464061333957, + "tokens_seen": 1203634176 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208726178535607, + "loss": 2.6157, + "theoretical_loss": 3.5864461092629343, + "tokens_seen": 1203699712 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032086258776328985, + "loss": 2.8626, + "theoretical_loss": 3.586428158442952, + "tokens_seen": 1203765248 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208525576730191, + "loss": 2.3889, + "theoretical_loss": 3.5864102088738536, + "tokens_seen": 1203830784 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208425275827482, + "loss": 2.8346, + "theoretical_loss": 3.586392260555485, + "tokens_seen": 1203896320 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032083249749247745, + "loss": 2.9258, + "theoretical_loss": 3.5863743134876906, + "tokens_seen": 1203961856 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032082246740220663, + "loss": 2.5813, + "theoretical_loss": 3.586356367670315, + "tokens_seen": 1204027392 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003208124373119358, + "loss": 2.7728, + "theoretical_loss": 3.5863384231032036, + "tokens_seen": 1204092928 + }, + { + "epoch": 0.36, + "learning_rate": 0.000320802407221665, + "loss": 2.6154, + "theoretical_loss": 3.5863204797862003, + "tokens_seen": 1204158464 + }, + { + "epoch": 0.36, + "objective/train/docs_used": 667638, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.698781728744507, + "objective/train/theoretical_loss": 3.586302537719151, + "objective/train/tokens_used": 1224684000, + "theoretical_loss": 3.586302537719151, + "tokens_seen": 1204224000 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207923771313942, + "loss": 2.659, + "theoretical_loss": 3.586302537719151, + "tokens_seen": 1204224000 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032078234704112336, + "loss": 2.6391, + "theoretical_loss": 3.5862845969018995, + "tokens_seen": 1204289536 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207723169508526, + "loss": 2.5357, + "theoretical_loss": 3.586266657334292, + "tokens_seen": 1204355072 + }, + { + "epoch": 0.36, + "learning_rate": 0.0003207622868605817, + "loss": 2.5632, + "theoretical_loss": 3.5862487190161723, + "tokens_seen": 1204420608 + }, + { + "epoch": 0.36, + "learning_rate": 0.00032075225677031095, + "loss": 2.7713, + "theoretical_loss": 3.5862307819473864, + "tokens_seen": 1204486144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207422266800401, + "loss": 2.5229, + "theoretical_loss": 3.5862128461277782, + "tokens_seen": 1204551680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207321965897693, + "loss": 2.493, + "theoretical_loss": 3.586194911557194, + "tokens_seen": 1204617216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207221664994985, + "loss": 2.8192, + "theoretical_loss": 3.586176978235478, + "tokens_seen": 1204682752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003207121364092277, + "loss": 2.8019, + "theoretical_loss": 3.586159046162476, + "tokens_seen": 1204748288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032070210631895686, + "loss": 2.4404, + "theoretical_loss": 3.586141115338032, + "tokens_seen": 1204813824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206920762286861, + "loss": 2.7097, + "theoretical_loss": 3.586123185761992, + "tokens_seen": 1204879360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206820461384152, + "loss": 2.8557, + "theoretical_loss": 3.586105257434201, + "tokens_seen": 1204944896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032067201604814446, + "loss": 2.8357, + "theoretical_loss": 3.5860873303545047, + "tokens_seen": 1205010432 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206619859578736, + "loss": 2.8592, + "theoretical_loss": 3.586069404522747, + "tokens_seen": 1205075968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206519558676028, + "loss": 2.8587, + "theoretical_loss": 3.586051479938775, + "tokens_seen": 1205141504 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320641925777332, + "loss": 2.7109, + "theoretical_loss": 3.5860335566024326, + "tokens_seen": 1205207040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206318956870612, + "loss": 2.5293, + "theoretical_loss": 3.586015634513566, + "tokens_seen": 1205272576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032062186559679036, + "loss": 2.7344, + "theoretical_loss": 3.585997713672019, + "tokens_seen": 1205338112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032061183550651954, + "loss": 2.5098, + "theoretical_loss": 3.5859797940776392, + "tokens_seen": 1205403648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003206018054162487, + "loss": 2.7707, + "theoretical_loss": 3.5859618757302707, + "tokens_seen": 1205469184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032059177532597796, + "loss": 2.7907, + "theoretical_loss": 3.585943958629759, + "tokens_seen": 1205534720 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205817452357071, + "loss": 2.6121, + "theoretical_loss": 3.5859260427759496, + "tokens_seen": 1205600256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205717151454363, + "loss": 2.8092, + "theoretical_loss": 3.5859081281686884, + "tokens_seen": 1205665792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205616850551655, + "loss": 2.4816, + "theoretical_loss": 3.5858902148078204, + "tokens_seen": 1205731328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205516549648947, + "loss": 2.4486, + "theoretical_loss": 3.5858723026931916, + "tokens_seen": 1205796864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 668120, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.765895366668701, + "objective/train/theoretical_loss": 3.585854391824647, + "objective/train/tokens_used": 1226322400, + "theoretical_loss": 3.585854391824647, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205416248746239, + "loss": 3.0271, + "theoretical_loss": 3.585854391824647, + "tokens_seen": 1205862400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032053159478435305, + "loss": 2.6968, + "theoretical_loss": 3.5858364822020325, + "tokens_seen": 1205927936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003205215646940823, + "loss": 2.7393, + "theoretical_loss": 3.585818573825194, + "tokens_seen": 1205993472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032051153460381146, + "loss": 2.6489, + "theoretical_loss": 3.585800666693977, + "tokens_seen": 1206059008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032050150451354064, + "loss": 2.6772, + "theoretical_loss": 3.585782760808227, + "tokens_seen": 1206124544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204914744232698, + "loss": 2.5699, + "theoretical_loss": 3.5857648561677893, + "tokens_seen": 1206190080 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320481444332999, + "loss": 2.8226, + "theoretical_loss": 3.585746952772511, + "tokens_seen": 1206255616 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204714142427282, + "loss": 2.678, + "theoretical_loss": 3.5857290506222363, + "tokens_seen": 1206321152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204613841524574, + "loss": 2.6826, + "theoretical_loss": 3.585711149716812, + "tokens_seen": 1206386688 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032045135406218655, + "loss": 2.725, + "theoretical_loss": 3.5856932500560834, + "tokens_seen": 1206452224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204413239719158, + "loss": 2.7681, + "theoretical_loss": 3.585675351639897, + "tokens_seen": 1206517760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204312938816449, + "loss": 2.7728, + "theoretical_loss": 3.5856574544680977, + "tokens_seen": 1206583296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032042126379137415, + "loss": 2.6691, + "theoretical_loss": 3.5856395585405325, + "tokens_seen": 1206648832 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032041123370110333, + "loss": 2.6117, + "theoretical_loss": 3.5856216638570464, + "tokens_seen": 1206714368 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003204012036108325, + "loss": 2.7022, + "theoretical_loss": 3.5856037704174857, + "tokens_seen": 1206779904 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203911735205617, + "loss": 2.757, + "theoretical_loss": 3.5855858782216963, + "tokens_seen": 1206845440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203811434302909, + "loss": 2.7985, + "theoretical_loss": 3.585567987269525, + "tokens_seen": 1206910976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032037111334002005, + "loss": 2.3973, + "theoretical_loss": 3.5855500975608163, + "tokens_seen": 1206976512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203610832497493, + "loss": 2.6385, + "theoretical_loss": 3.5855322090954176, + "tokens_seen": 1207042048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203510531594784, + "loss": 2.6184, + "theoretical_loss": 3.585514321873174, + "tokens_seen": 1207107584 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032034102306920765, + "loss": 2.8333, + "theoretical_loss": 3.5854964358939325, + "tokens_seen": 1207173120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032033099297893683, + "loss": 2.6203, + "theoretical_loss": 3.5854785511575393, + "tokens_seen": 1207238656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032033099297893683, + "loss": 2.6739, + "theoretical_loss": 3.5854606676638396, + "tokens_seen": 1207304192 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320320962888666, + "loss": 2.7132, + "theoretical_loss": 3.5854427854126802, + "tokens_seen": 1207369728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203109327983952, + "loss": 2.6528, + "theoretical_loss": 3.585424904403908, + "tokens_seen": 1207435264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 669193, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.440397024154663, + "objective/train/theoretical_loss": 3.5854070246373677, + "objective/train/tokens_used": 1227960800, + "theoretical_loss": 3.5854070246373677, + "tokens_seen": 1207500800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003203009027081244, + "loss": 2.4402, + "theoretical_loss": 3.5854070246373677, + "tokens_seen": 1207500800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032029087261785356, + "loss": 2.8087, + "theoretical_loss": 3.585389146112907, + "tokens_seen": 1207566336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202808425275828, + "loss": 2.79, + "theoretical_loss": 3.5853712688303707, + "tokens_seen": 1207631872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202708124373119, + "loss": 2.7522, + "theoretical_loss": 3.585353392789607, + "tokens_seen": 1207697408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032026078234704115, + "loss": 2.6718, + "theoretical_loss": 3.585335517990461, + "tokens_seen": 1207762944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202507522567703, + "loss": 2.6665, + "theoretical_loss": 3.58531764443278, + "tokens_seen": 1207828480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202407221664995, + "loss": 2.4913, + "theoretical_loss": 3.585299772116409, + "tokens_seen": 1207894016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202306920762287, + "loss": 2.8258, + "theoretical_loss": 3.5852819010411956, + "tokens_seen": 1207959552 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202206619859579, + "loss": 2.7235, + "theoretical_loss": 3.5852640312069863, + "tokens_seen": 1208025088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032021063189568706, + "loss": 2.5067, + "theoretical_loss": 3.585246162613627, + "tokens_seen": 1208090624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003202006018054163, + "loss": 2.7812, + "theoretical_loss": 3.585228295260965, + "tokens_seen": 1208156160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201905717151454, + "loss": 2.7972, + "theoretical_loss": 3.5852104291488462, + "tokens_seen": 1208221696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032018054162487466, + "loss": 2.5162, + "theoretical_loss": 3.5851925642771167, + "tokens_seen": 1208287232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201705115346038, + "loss": 2.7096, + "theoretical_loss": 3.5851747006456245, + "tokens_seen": 1208352768 + }, + { + "epoch": 0.37, + "learning_rate": 0.000320160481444333, + "loss": 2.7143, + "theoretical_loss": 3.585156838254216, + "tokens_seen": 1208418304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201504513540622, + "loss": 2.818, + "theoretical_loss": 3.585138977102736, + "tokens_seen": 1208483840 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201404212637914, + "loss": 2.6218, + "theoretical_loss": 3.5851211171910338, + "tokens_seen": 1208549376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032013039117352056, + "loss": 2.395, + "theoretical_loss": 3.5851032585189544, + "tokens_seen": 1208614912 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032012036108324974, + "loss": 2.3369, + "theoretical_loss": 3.585085401086345, + "tokens_seen": 1208680448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003201103309929789, + "loss": 2.5753, + "theoretical_loss": 3.585067544893053, + "tokens_seen": 1208745984 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032010030090270816, + "loss": 2.8804, + "theoretical_loss": 3.5850496899389244, + "tokens_seen": 1208811520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200902708124373, + "loss": 2.6475, + "theoretical_loss": 3.5850318362238056, + "tokens_seen": 1208877056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200802407221665, + "loss": 2.8044, + "theoretical_loss": 3.5850139837475448, + "tokens_seen": 1208942592 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032007021063189565, + "loss": 2.5214, + "theoretical_loss": 3.584996132509988, + "tokens_seen": 1209008128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200601805416249, + "loss": 2.7903, + "theoretical_loss": 3.5849782825109826, + "tokens_seen": 1209073664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 670409, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.449265480041504, + "objective/train/theoretical_loss": 3.584960433750375, + "objective/train/tokens_used": 1229599200, + "theoretical_loss": 3.584960433750375, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032005015045135407, + "loss": 2.629, + "theoretical_loss": 3.584960433750375, + "tokens_seen": 1209139200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032004012036108325, + "loss": 2.6543, + "theoretical_loss": 3.5849425862280127, + "tokens_seen": 1209204736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032003009027081243, + "loss": 2.5549, + "theoretical_loss": 3.5849247399437423, + "tokens_seen": 1209270272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032002006018054166, + "loss": 2.7935, + "theoretical_loss": 3.5849068948974114, + "tokens_seen": 1209335808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003200100300902708, + "loss": 2.693, + "theoretical_loss": 3.584889051088866, + "tokens_seen": 1209401344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00032, + "loss": 2.8575, + "theoretical_loss": 3.5848712085179537, + "tokens_seen": 1209466880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031998996990972915, + "loss": 2.7422, + "theoretical_loss": 3.584853367184522, + "tokens_seen": 1209532416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199799398194584, + "loss": 2.593, + "theoretical_loss": 3.584835527088418, + "tokens_seen": 1209597952 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031996990972918757, + "loss": 2.6048, + "theoretical_loss": 3.584817688229488, + "tokens_seen": 1209663488 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031995987963891675, + "loss": 2.7282, + "theoretical_loss": 3.58479985060758, + "tokens_seen": 1209729024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031994984954864593, + "loss": 2.7549, + "theoretical_loss": 3.584782014222541, + "tokens_seen": 1209794560 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199398194583751, + "loss": 2.4356, + "theoretical_loss": 3.5847641790742184, + "tokens_seen": 1209860096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003199297893681043, + "loss": 2.677, + "theoretical_loss": 3.584746345162459, + "tokens_seen": 1209925632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031991975927783353, + "loss": 2.7683, + "theoretical_loss": 3.5847285124871107, + "tokens_seen": 1209991168 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031990972918756266, + "loss": 2.6618, + "theoretical_loss": 3.58471068104802, + "tokens_seen": 1210056704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198996990972919, + "loss": 2.6729, + "theoretical_loss": 3.5846928508450353, + "tokens_seen": 1210122240 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319889669007021, + "loss": 2.5604, + "theoretical_loss": 3.5846750218780032, + "tokens_seen": 1210187776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031987963891675025, + "loss": 2.5945, + "theoretical_loss": 3.584657194146771, + "tokens_seen": 1210253312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031986960882647943, + "loss": 2.5825, + "theoretical_loss": 3.584639367651187, + "tokens_seen": 1210318848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198595787362086, + "loss": 2.6042, + "theoretical_loss": 3.5846215423910976, + "tokens_seen": 1210384384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198495486459378, + "loss": 2.6627, + "theoretical_loss": 3.5846037183663513, + "tokens_seen": 1210449920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031983951855566703, + "loss": 2.6505, + "theoretical_loss": 3.5845858955767946, + "tokens_seen": 1210515456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031982948846539616, + "loss": 2.5319, + "theoretical_loss": 3.5845680740222763, + "tokens_seen": 1210580992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198194583751254, + "loss": 2.6629, + "theoretical_loss": 3.584550253702642, + "tokens_seen": 1210646528 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003198094282848546, + "loss": 2.6587, + "theoretical_loss": 3.5845324346177416, + "tokens_seen": 1210712064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 670966, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8296167850494385, + "objective/train/theoretical_loss": 3.584514616767421, + "objective/train/tokens_used": 1231237600, + "theoretical_loss": 3.584514616767421, + "tokens_seen": 1210777600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031979939819458376, + "loss": 2.6485, + "theoretical_loss": 3.584514616767421, + "tokens_seen": 1210777600 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319789368104313, + "loss": 2.6005, + "theoretical_loss": 3.5844968001515287, + "tokens_seen": 1210843136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197793380140421, + "loss": 2.715, + "theoretical_loss": 3.584478984769912, + "tokens_seen": 1210908672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031976930792377135, + "loss": 2.689, + "theoretical_loss": 3.584461170622419, + "tokens_seen": 1210974208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197592778335005, + "loss": 2.8714, + "theoretical_loss": 3.584443357708897, + "tokens_seen": 1211039744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197492477432297, + "loss": 2.2651, + "theoretical_loss": 3.584425546029194, + "tokens_seen": 1211105280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197392176529589, + "loss": 2.6919, + "theoretical_loss": 3.5844077355831576, + "tokens_seen": 1211170816 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197291875626881, + "loss": 2.6406, + "theoretical_loss": 3.584389926370635, + "tokens_seen": 1211236352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031971915747241726, + "loss": 2.7218, + "theoretical_loss": 3.584372118391476, + "tokens_seen": 1211301888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003197091273821465, + "loss": 2.6949, + "theoretical_loss": 3.5843543116455265, + "tokens_seen": 1211367424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196990972918756, + "loss": 2.5578, + "theoretical_loss": 3.584336506132635, + "tokens_seen": 1211432960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031968906720160486, + "loss": 2.6447, + "theoretical_loss": 3.5843187018526494, + "tokens_seen": 1211498496 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319679037111334, + "loss": 2.7112, + "theoretical_loss": 3.5843008988054184, + "tokens_seen": 1211564032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196690070210632, + "loss": 2.6566, + "theoretical_loss": 3.584283096990789, + "tokens_seen": 1211629568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196589769307924, + "loss": 2.6208, + "theoretical_loss": 3.5842652964086095, + "tokens_seen": 1211695104 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196489468405216, + "loss": 2.5844, + "theoretical_loss": 3.584247497058728, + "tokens_seen": 1211760640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031963891675025076, + "loss": 2.6369, + "theoretical_loss": 3.5842296989409923, + "tokens_seen": 1211826176 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031962888665997994, + "loss": 2.7917, + "theoretical_loss": 3.5842119020552508, + "tokens_seen": 1211891712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003196188565697091, + "loss": 2.7858, + "theoretical_loss": 3.584194106401351, + "tokens_seen": 1211957248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031960882647943836, + "loss": 2.6762, + "theoretical_loss": 3.5841763119791423, + "tokens_seen": 1212022784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195987963891675, + "loss": 2.9897, + "theoretical_loss": 3.5841585187884712, + "tokens_seen": 1212088320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195887662988967, + "loss": 2.5801, + "theoretical_loss": 3.5841407268291876, + "tokens_seen": 1212153856 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031957873620862585, + "loss": 2.7895, + "theoretical_loss": 3.5841229361011386, + "tokens_seen": 1212219392 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195687061183551, + "loss": 2.7946, + "theoretical_loss": 3.5841051466041725, + "tokens_seen": 1212284928 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031955867602808427, + "loss": 2.6184, + "theoretical_loss": 3.5840873583381376, + "tokens_seen": 1212350464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 672128, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6712498664855957, + "objective/train/theoretical_loss": 3.5840695713028827, + "objective/train/tokens_used": 1232876000, + "theoretical_loss": 3.5840695713028827, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031954864593781345, + "loss": 2.5049, + "theoretical_loss": 3.5840695713028827, + "tokens_seen": 1212416000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031953861584754263, + "loss": 2.7097, + "theoretical_loss": 3.5840517854982554, + "tokens_seen": 1212481536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031952858575727186, + "loss": 2.6616, + "theoretical_loss": 3.584034000924105, + "tokens_seen": 1212547072 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319518555667001, + "loss": 2.4007, + "theoretical_loss": 3.584016217580279, + "tokens_seen": 1212612608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003195085255767302, + "loss": 2.7693, + "theoretical_loss": 3.583998435466626, + "tokens_seen": 1212678144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031949849548645935, + "loss": 2.7655, + "theoretical_loss": 3.5839806545829944, + "tokens_seen": 1212743680 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194884653961886, + "loss": 2.6329, + "theoretical_loss": 3.583962874929233, + "tokens_seen": 1212809216 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031947843530591777, + "loss": 2.5438, + "theoretical_loss": 3.5839450965051904, + "tokens_seen": 1212874752 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031946840521564695, + "loss": 2.6286, + "theoretical_loss": 3.583927319310714, + "tokens_seen": 1212940288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031945837512537613, + "loss": 2.411, + "theoretical_loss": 3.5839095433456536, + "tokens_seen": 1213005824 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194483450351053, + "loss": 2.4915, + "theoretical_loss": 3.583891768609857, + "tokens_seen": 1213071360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194383149448345, + "loss": 2.588, + "theoretical_loss": 3.5838739951031737, + "tokens_seen": 1213136896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031942828485456373, + "loss": 2.5782, + "theoretical_loss": 3.5838562228254514, + "tokens_seen": 1213202432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031941825476429286, + "loss": 2.6396, + "theoretical_loss": 3.5838384517765385, + "tokens_seen": 1213267968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003194082246740221, + "loss": 2.6197, + "theoretical_loss": 3.5838206819562846, + "tokens_seen": 1213333504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193981945837512, + "loss": 2.762, + "theoretical_loss": 3.5838029133645377, + "tokens_seen": 1213399040 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031938816449348045, + "loss": 2.6373, + "theoretical_loss": 3.583785146001147, + "tokens_seen": 1213464576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031937813440320964, + "loss": 2.6973, + "theoretical_loss": 3.583767379865961, + "tokens_seen": 1213530112 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193681043129388, + "loss": 2.5355, + "theoretical_loss": 3.5837496149588284, + "tokens_seen": 1213595648 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319358074222668, + "loss": 2.7368, + "theoretical_loss": 3.583731851279598, + "tokens_seen": 1213661184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031934804413239723, + "loss": 2.8614, + "theoretical_loss": 3.583714088828119, + "tokens_seen": 1213726720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031933801404212636, + "loss": 2.5393, + "theoretical_loss": 3.5836963276042404, + "tokens_seen": 1213792256 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193279839518556, + "loss": 2.7673, + "theoretical_loss": 3.58367856760781, + "tokens_seen": 1213857792 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003193179538615847, + "loss": 2.7873, + "theoretical_loss": 3.583660808838678, + "tokens_seen": 1213923328 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031930792377131396, + "loss": 2.758, + "theoretical_loss": 3.5836430512966926, + "tokens_seen": 1213988864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 672740, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.2003424167633057, + "objective/train/theoretical_loss": 3.5836252949817027, + "objective/train/tokens_used": 1234514400, + "theoretical_loss": 3.5836252949817027, + "tokens_seen": 1214054400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031929789368104314, + "loss": 2.7173, + "theoretical_loss": 3.5836252949817027, + "tokens_seen": 1214054400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192878635907723, + "loss": 2.7391, + "theoretical_loss": 3.583607539893557, + "tokens_seen": 1214119936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192778335005015, + "loss": 2.6003, + "theoretical_loss": 3.583589786032106, + "tokens_seen": 1214185472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192678034102307, + "loss": 2.8058, + "theoretical_loss": 3.583572033397197, + "tokens_seen": 1214251008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031925777331995986, + "loss": 2.8194, + "theoretical_loss": 3.5835542819886808, + "tokens_seen": 1214316544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192477432296891, + "loss": 2.5911, + "theoretical_loss": 3.5835365318064047, + "tokens_seen": 1214382080 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192377131394182, + "loss": 2.6463, + "theoretical_loss": 3.5835187828502186, + "tokens_seen": 1214447616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031922768304914746, + "loss": 2.602, + "theoretical_loss": 3.583501035119972, + "tokens_seen": 1214513152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192176529588766, + "loss": 2.4129, + "theoretical_loss": 3.583483288615514, + "tokens_seen": 1214578688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003192076228686058, + "loss": 2.7214, + "theoretical_loss": 3.5834655433366933, + "tokens_seen": 1214644224 + }, + { + "epoch": 0.37, + "learning_rate": 0.000319197592778335, + "loss": 2.5718, + "theoretical_loss": 3.583447799283359, + "tokens_seen": 1214709760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191875626880642, + "loss": 2.8878, + "theoretical_loss": 3.583430056455361, + "tokens_seen": 1214775296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031917753259779337, + "loss": 2.3432, + "theoretical_loss": 3.5834123148525485, + "tokens_seen": 1214840832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191675025075226, + "loss": 2.7502, + "theoretical_loss": 3.583394574474771, + "tokens_seen": 1214906368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031915747241725173, + "loss": 2.6352, + "theoretical_loss": 3.5833768353218773, + "tokens_seen": 1214971904 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031914744232698096, + "loss": 2.4479, + "theoretical_loss": 3.583359097393717, + "tokens_seen": 1215037440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191374122367101, + "loss": 2.6745, + "theoretical_loss": 3.583341360690139, + "tokens_seen": 1215102976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191273821464393, + "loss": 2.7993, + "theoretical_loss": 3.5833236252109937, + "tokens_seen": 1215168512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191173520561685, + "loss": 2.6827, + "theoretical_loss": 3.5833058909561295, + "tokens_seen": 1215234048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003191073219658977, + "loss": 2.8311, + "theoretical_loss": 3.583288157925397, + "tokens_seen": 1215299584 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031909729187562687, + "loss": 2.8424, + "theoretical_loss": 3.583270426118645, + "tokens_seen": 1215365120 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031908726178535605, + "loss": 2.927, + "theoretical_loss": 3.583252695535723, + "tokens_seen": 1215430656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031907723169508523, + "loss": 2.7119, + "theoretical_loss": 3.58323496617648, + "tokens_seen": 1215496192 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031906720160481447, + "loss": 2.7552, + "theoretical_loss": 3.583217238040767, + "tokens_seen": 1215561728 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031905717151454365, + "loss": 2.6251, + "theoretical_loss": 3.583199511128433, + "tokens_seen": 1215627264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 673834, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.846170425415039, + "objective/train/theoretical_loss": 3.5831817854393266, + "objective/train/tokens_used": 1236152800, + "theoretical_loss": 3.5831817854393266, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031904714142427283, + "loss": 2.8292, + "theoretical_loss": 3.5831817854393266, + "tokens_seen": 1215692800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031903711133400206, + "loss": 2.6614, + "theoretical_loss": 3.5831640609732993, + "tokens_seen": 1215758336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190270812437312, + "loss": 2.634, + "theoretical_loss": 3.5831463377301995, + "tokens_seen": 1215823872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003190170511534604, + "loss": 2.6109, + "theoretical_loss": 3.5831286157098767, + "tokens_seen": 1215889408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031900702106318955, + "loss": 2.6929, + "theoretical_loss": 3.5831108949121817, + "tokens_seen": 1215954944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189969909729188, + "loss": 2.7014, + "theoretical_loss": 3.5830931753369635, + "tokens_seen": 1216020480 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031898696088264797, + "loss": 2.7554, + "theoretical_loss": 3.5830754569840724, + "tokens_seen": 1216086016 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031897693079237715, + "loss": 2.3739, + "theoretical_loss": 3.5830577398533574, + "tokens_seen": 1216151552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031896690070210633, + "loss": 2.8485, + "theoretical_loss": 3.5830400239446694, + "tokens_seen": 1216217088 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189568706118355, + "loss": 2.9264, + "theoretical_loss": 3.5830223092578573, + "tokens_seen": 1216282624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189468405215647, + "loss": 3.0185, + "theoretical_loss": 3.583004595792772, + "tokens_seen": 1216348160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031893681043129393, + "loss": 2.384, + "theoretical_loss": 3.582986883549262, + "tokens_seen": 1216413696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031892678034102306, + "loss": 2.6403, + "theoretical_loss": 3.5829691725271786, + "tokens_seen": 1216479232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189167502507523, + "loss": 2.6133, + "theoretical_loss": 3.5829514627263714, + "tokens_seen": 1216544768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003189067201604814, + "loss": 2.5405, + "theoretical_loss": 3.5829337541466906, + "tokens_seen": 1216610304 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031889669007021065, + "loss": 2.6433, + "theoretical_loss": 3.5829160467879855, + "tokens_seen": 1216675840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031888665997993984, + "loss": 2.6715, + "theoretical_loss": 3.5828983406501065, + "tokens_seen": 1216741376 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318876629889669, + "loss": 2.6068, + "theoretical_loss": 3.582880635732904, + "tokens_seen": 1216806912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188665997993982, + "loss": 2.4911, + "theoretical_loss": 3.5828629320362273, + "tokens_seen": 1216872448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031885656970912743, + "loss": 2.6389, + "theoretical_loss": 3.5828452295599273, + "tokens_seen": 1216937984 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031884653961885656, + "loss": 2.7881, + "theoretical_loss": 3.5828275283038544, + "tokens_seen": 1217003520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188365095285858, + "loss": 2.6524, + "theoretical_loss": 3.582809828267858, + "tokens_seen": 1217069056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003188264794383149, + "loss": 2.6268, + "theoretical_loss": 3.582792129451789, + "tokens_seen": 1217134592 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031881644934804416, + "loss": 2.6762, + "theoretical_loss": 3.582774431855497, + "tokens_seen": 1217200128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031880641925777334, + "loss": 2.2686, + "theoretical_loss": 3.5827567354788323, + "tokens_seen": 1217265664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 674331, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5128471851348877, + "objective/train/theoretical_loss": 3.5827390403216457, + "objective/train/tokens_used": 1237791200, + "theoretical_loss": 3.5827390403216457, + "tokens_seen": 1217331200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187963891675025, + "loss": 2.7099, + "theoretical_loss": 3.5827390403216457, + "tokens_seen": 1217331200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187863590772317, + "loss": 2.8773, + "theoretical_loss": 3.5827213463837877, + "tokens_seen": 1217396736 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187763289869609, + "loss": 2.7363, + "theoretical_loss": 3.5827036536651073, + "tokens_seen": 1217462272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031876629889669006, + "loss": 2.6986, + "theoretical_loss": 3.5826859621654568, + "tokens_seen": 1217527808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187562688064193, + "loss": 2.4538, + "theoretical_loss": 3.582668271884685, + "tokens_seen": 1217593344 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187462387161484, + "loss": 2.6988, + "theoretical_loss": 3.582650582822643, + "tokens_seen": 1217658880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031873620862587766, + "loss": 2.6411, + "theoretical_loss": 3.5826328949791812, + "tokens_seen": 1217724416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187261785356068, + "loss": 2.4917, + "theoretical_loss": 3.58261520835415, + "tokens_seen": 1217789952 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318716148445336, + "loss": 2.7464, + "theoretical_loss": 3.5825975229474, + "tokens_seen": 1217855488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003187061183550652, + "loss": 2.669, + "theoretical_loss": 3.582579838758782, + "tokens_seen": 1217921024 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186960882647944, + "loss": 2.4795, + "theoretical_loss": 3.5825621557881457, + "tokens_seen": 1217986560 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031868605817452357, + "loss": 2.6991, + "theoretical_loss": 3.5825444740353425, + "tokens_seen": 1218052096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186760280842528, + "loss": 2.6764, + "theoretical_loss": 3.5825267935002225, + "tokens_seen": 1218117632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031866599799398193, + "loss": 2.5941, + "theoretical_loss": 3.5825091141826366, + "tokens_seen": 1218183168 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031865596790371116, + "loss": 2.6048, + "theoretical_loss": 3.582491436082436, + "tokens_seen": 1218248704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186459378134403, + "loss": 2.5775, + "theoretical_loss": 3.58247375919947, + "tokens_seen": 1218314240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186359077231695, + "loss": 2.7505, + "theoretical_loss": 3.582456083533591, + "tokens_seen": 1218379776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186258776328987, + "loss": 2.2671, + "theoretical_loss": 3.582438409084648, + "tokens_seen": 1218445312 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003186158475426279, + "loss": 2.9417, + "theoretical_loss": 3.5824207358524935, + "tokens_seen": 1218510848 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031860581745235707, + "loss": 2.6654, + "theoretical_loss": 3.582403063836977, + "tokens_seen": 1218576384 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031859578736208625, + "loss": 2.6196, + "theoretical_loss": 3.5823853930379497, + "tokens_seen": 1218641920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031858575727181543, + "loss": 2.7017, + "theoretical_loss": 3.5823677234552624, + "tokens_seen": 1218707456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031857572718154467, + "loss": 2.6233, + "theoretical_loss": 3.5823500550887664, + "tokens_seen": 1218772992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185656970912738, + "loss": 2.3861, + "theoretical_loss": 3.5823323879383118, + "tokens_seen": 1218838528 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031855566700100303, + "loss": 2.766, + "theoretical_loss": 3.5823147220037503, + "tokens_seen": 1218904064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 675627, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.326138973236084, + "objective/train/theoretical_loss": 3.582297057284933, + "objective/train/tokens_used": 1239429600, + "theoretical_loss": 3.582297057284933, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185456369107322, + "loss": 2.6397, + "theoretical_loss": 3.582297057284933, + "tokens_seen": 1218969600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003185356068204614, + "loss": 2.5176, + "theoretical_loss": 3.58227939378171, + "tokens_seen": 1219035136 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031852557673019057, + "loss": 2.5311, + "theoretical_loss": 3.5822617314939325, + "tokens_seen": 1219100672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031851554663991975, + "loss": 2.6315, + "theoretical_loss": 3.582244070421452, + "tokens_seen": 1219166208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031850551654964893, + "loss": 2.5092, + "theoretical_loss": 3.5822264105641195, + "tokens_seen": 1219231744 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031849548645937817, + "loss": 2.5286, + "theoretical_loss": 3.582208751921786, + "tokens_seen": 1219297280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184854563691073, + "loss": 2.8103, + "theoretical_loss": 3.582191094494302, + "tokens_seen": 1219362816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031847542627883653, + "loss": 2.4883, + "theoretical_loss": 3.5821734382815196, + "tokens_seen": 1219428352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031846539618856566, + "loss": 2.7542, + "theoretical_loss": 3.582155783283289, + "tokens_seen": 1219493888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184553660982949, + "loss": 2.481, + "theoretical_loss": 3.5821381294994628, + "tokens_seen": 1219559424 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184453360080241, + "loss": 2.7431, + "theoretical_loss": 3.582120476929891, + "tokens_seen": 1219624960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031843530591775326, + "loss": 2.4548, + "theoretical_loss": 3.5821028255744247, + "tokens_seen": 1219690496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031842527582748244, + "loss": 2.6645, + "theoretical_loss": 3.582085175432916, + "tokens_seen": 1219756032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184152457372116, + "loss": 2.363, + "theoretical_loss": 3.5820675265052158, + "tokens_seen": 1219821568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003184052156469408, + "loss": 2.6981, + "theoretical_loss": 3.5820498787911754, + "tokens_seen": 1219887104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031839518555667004, + "loss": 2.688, + "theoretical_loss": 3.582032232290646, + "tokens_seen": 1219952640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031838515546639916, + "loss": 2.6851, + "theoretical_loss": 3.5820145870034796, + "tokens_seen": 1220018176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183751253761284, + "loss": 2.6927, + "theoretical_loss": 3.5819969429295266, + "tokens_seen": 1220083712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183650952858576, + "loss": 2.7519, + "theoretical_loss": 3.5819793000686397, + "tokens_seen": 1220149248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031835506519558676, + "loss": 2.5282, + "theoretical_loss": 3.581961658420669, + "tokens_seen": 1220214784 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031834503510531594, + "loss": 2.6981, + "theoretical_loss": 3.5819440179854674, + "tokens_seen": 1220280320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183350050150451, + "loss": 2.4955, + "theoretical_loss": 3.5819263787628848, + "tokens_seen": 1220345856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183249749247743, + "loss": 2.6698, + "theoretical_loss": 3.5819087407527737, + "tokens_seen": 1220411392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031831494483450354, + "loss": 2.8391, + "theoretical_loss": 3.5818911039549857, + "tokens_seen": 1220476928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003183049147442327, + "loss": 2.6485, + "theoretical_loss": 3.581873468369372, + "tokens_seen": 1220542464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 676618, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6377201080322266, + "objective/train/theoretical_loss": 3.5818558339957844, + "objective/train/tokens_used": 1241068000, + "theoretical_loss": 3.5818558339957844, + "tokens_seen": 1220608000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182948846539619, + "loss": 2.5538, + "theoretical_loss": 3.5818558339957844, + "tokens_seen": 1220608000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182848545636911, + "loss": 2.7519, + "theoretical_loss": 3.581838200834075, + "tokens_seen": 1220673536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031827482447342026, + "loss": 2.6153, + "theoretical_loss": 3.581820568884094, + "tokens_seen": 1220739072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182647943831495, + "loss": 2.7711, + "theoretical_loss": 3.581802938145695, + "tokens_seen": 1220804608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182547642928786, + "loss": 2.7786, + "theoretical_loss": 3.581785308618729, + "tokens_seen": 1220870144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031824473420260786, + "loss": 2.5908, + "theoretical_loss": 3.5817676803030465, + "tokens_seen": 1220935680 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318234704112337, + "loss": 2.6733, + "theoretical_loss": 3.5817500531985007, + "tokens_seen": 1221001216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182246740220662, + "loss": 2.5456, + "theoretical_loss": 3.581732427304943, + "tokens_seen": 1221066752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182146439317954, + "loss": 2.5502, + "theoretical_loss": 3.5817148026222254, + "tokens_seen": 1221132288 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003182046138415246, + "loss": 2.8477, + "theoretical_loss": 3.5816971791501993, + "tokens_seen": 1221197824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031819458375125377, + "loss": 2.6389, + "theoretical_loss": 3.5816795568887168, + "tokens_seen": 1221263360 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318184553660983, + "loss": 2.6474, + "theoretical_loss": 3.58166193583763, + "tokens_seen": 1221328896 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031817452357071213, + "loss": 2.6726, + "theoretical_loss": 3.5816443159967903, + "tokens_seen": 1221394432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031816449348044136, + "loss": 2.7162, + "theoretical_loss": 3.58162669736605, + "tokens_seen": 1221459968 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181544633901705, + "loss": 2.629, + "theoretical_loss": 3.5816090799452613, + "tokens_seen": 1221525504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181444332998997, + "loss": 2.5372, + "theoretical_loss": 3.581591463734276, + "tokens_seen": 1221591040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181344032096289, + "loss": 2.5658, + "theoretical_loss": 3.581573848732946, + "tokens_seen": 1221656576 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003181243731193581, + "loss": 2.5672, + "theoretical_loss": 3.581556234941123, + "tokens_seen": 1221722112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031811434302908727, + "loss": 2.8258, + "theoretical_loss": 3.58153862235866, + "tokens_seen": 1221787648 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031810431293881645, + "loss": 2.8054, + "theoretical_loss": 3.581521010985408, + "tokens_seen": 1221853184 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031809428284854563, + "loss": 2.7113, + "theoretical_loss": 3.5815034008212203, + "tokens_seen": 1221918720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031808425275827487, + "loss": 2.6263, + "theoretical_loss": 3.5814857918659477, + "tokens_seen": 1221984256 + }, + { + "epoch": 0.37, + "learning_rate": 0.000318074222668004, + "loss": 2.9308, + "theoretical_loss": 3.5814681841194442, + "tokens_seen": 1222049792 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031806419257773323, + "loss": 2.6379, + "theoretical_loss": 3.58145057758156, + "tokens_seen": 1222115328 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180541624874624, + "loss": 2.5758, + "theoretical_loss": 3.581432972252149, + "tokens_seen": 1222180864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 677215, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.3762056827545166, + "objective/train/theoretical_loss": 3.5814153681310623, + "objective/train/tokens_used": 1242706400, + "theoretical_loss": 3.5814153681310623, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003180441323971916, + "loss": 2.7435, + "theoretical_loss": 3.5814153681310623, + "tokens_seen": 1222246400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031803410230692077, + "loss": 2.716, + "theoretical_loss": 3.581397765218153, + "tokens_seen": 1222311936 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031802407221664995, + "loss": 2.6065, + "theoretical_loss": 3.581380163513273, + "tokens_seen": 1222377472 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031801404212637913, + "loss": 2.7569, + "theoretical_loss": 3.5813625630162744, + "tokens_seen": 1222443008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031800401203610837, + "loss": 2.7203, + "theoretical_loss": 3.58134496372701, + "tokens_seen": 1222508544 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179939819458375, + "loss": 2.6444, + "theoretical_loss": 3.581327365645332, + "tokens_seen": 1222574080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031798395185556673, + "loss": 2.5914, + "theoretical_loss": 3.581309768771093, + "tokens_seen": 1222639616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031797392176529586, + "loss": 2.6799, + "theoretical_loss": 3.581292173104145, + "tokens_seen": 1222705152 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179638916750251, + "loss": 2.8491, + "theoretical_loss": 3.581274578644341, + "tokens_seen": 1222770688 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179538615847543, + "loss": 2.4735, + "theoretical_loss": 3.5812569853915335, + "tokens_seen": 1222836224 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031794383149448346, + "loss": 2.7763, + "theoretical_loss": 3.5812393933455744, + "tokens_seen": 1222901760 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031793380140421264, + "loss": 2.55, + "theoretical_loss": 3.5812218025063167, + "tokens_seen": 1222967296 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003179237713139418, + "loss": 2.4893, + "theoretical_loss": 3.581204212873613, + "tokens_seen": 1223032832 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317913741223671, + "loss": 2.7899, + "theoretical_loss": 3.581186624447316, + "tokens_seen": 1223098368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031790371113340024, + "loss": 2.4937, + "theoretical_loss": 3.5811690372272773, + "tokens_seen": 1223163904 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031789368104312936, + "loss": 2.8664, + "theoretical_loss": 3.581151451213351, + "tokens_seen": 1223229440 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178836509528586, + "loss": 2.7589, + "theoretical_loss": 3.5811338664053887, + "tokens_seen": 1223294976 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178736208625878, + "loss": 2.4983, + "theoretical_loss": 3.581116282803244, + "tokens_seen": 1223360512 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031786359077231696, + "loss": 2.7072, + "theoretical_loss": 3.581098700406769, + "tokens_seen": 1223426048 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031785356068204614, + "loss": 2.6054, + "theoretical_loss": 3.581081119215816, + "tokens_seen": 1223491584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178435305917753, + "loss": 2.4573, + "theoretical_loss": 3.5810635392302395, + "tokens_seen": 1223557120 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178335005015045, + "loss": 2.5697, + "theoretical_loss": 3.5810459604498908, + "tokens_seen": 1223622656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031782347041123374, + "loss": 2.4891, + "theoretical_loss": 3.5810283828746226, + "tokens_seen": 1223688192 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031781344032096287, + "loss": 2.8321, + "theoretical_loss": 3.5810108065042887, + "tokens_seen": 1223753728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003178034102306921, + "loss": 2.7512, + "theoretical_loss": 3.5809932313387414, + "tokens_seen": 1223819264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 678133, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7944464683532715, + "objective/train/theoretical_loss": 3.5809756573778335, + "objective/train/tokens_used": 1244344800, + "theoretical_loss": 3.5809756573778335, + "tokens_seen": 1223884800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031779338014042123, + "loss": 2.6868, + "theoretical_loss": 3.5809756573778335, + "tokens_seen": 1223884800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031778335005015046, + "loss": 2.837, + "theoretical_loss": 3.5809580846214186, + "tokens_seen": 1223950336 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031777331995987964, + "loss": 2.7034, + "theoretical_loss": 3.580940513069349, + "tokens_seen": 1224015872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177632898696088, + "loss": 2.7941, + "theoretical_loss": 3.580922942721478, + "tokens_seen": 1224081408 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317753259779338, + "loss": 2.7049, + "theoretical_loss": 3.580905373577659, + "tokens_seen": 1224146944 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177432296890672, + "loss": 2.7759, + "theoretical_loss": 3.580887805637744, + "tokens_seen": 1224212480 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031773319959879637, + "loss": 2.6332, + "theoretical_loss": 3.5808702389015865, + "tokens_seen": 1224278016 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003177231695085256, + "loss": 2.7317, + "theoretical_loss": 3.5808526733690402, + "tokens_seen": 1224343552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031771313941825473, + "loss": 2.6885, + "theoretical_loss": 3.580835109039958, + "tokens_seen": 1224409088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031770310932798397, + "loss": 2.6962, + "theoretical_loss": 3.5808175459141918, + "tokens_seen": 1224474624 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031769307923771315, + "loss": 2.6085, + "theoretical_loss": 3.5807999839915965, + "tokens_seen": 1224540160 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031768304914744233, + "loss": 2.5963, + "theoretical_loss": 3.5807824232720247, + "tokens_seen": 1224605696 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176730190571715, + "loss": 2.7513, + "theoretical_loss": 3.580764863755329, + "tokens_seen": 1224671232 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176629889669007, + "loss": 2.6441, + "theoretical_loss": 3.580747305441363, + "tokens_seen": 1224736768 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031765295887662987, + "loss": 2.3998, + "theoretical_loss": 3.58072974832998, + "tokens_seen": 1224802304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176429287863591, + "loss": 2.7632, + "theoretical_loss": 3.580712192421034, + "tokens_seen": 1224867840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031763289869608823, + "loss": 2.6429, + "theoretical_loss": 3.580694637714377, + "tokens_seen": 1224933376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031762286860581747, + "loss": 2.5251, + "theoretical_loss": 3.5806770842098636, + "tokens_seen": 1224998912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003176128385155466, + "loss": 2.5689, + "theoretical_loss": 3.580659531907346, + "tokens_seen": 1225064448 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031760280842527583, + "loss": 2.8738, + "theoretical_loss": 3.5806419808066785, + "tokens_seen": 1225129984 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317592778335005, + "loss": 2.7404, + "theoretical_loss": 3.580624430907714, + "tokens_seen": 1225195520 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175827482447342, + "loss": 2.5556, + "theoretical_loss": 3.5806068822103065, + "tokens_seen": 1225261056 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175727181544634, + "loss": 2.8817, + "theoretical_loss": 3.580589334714309, + "tokens_seen": 1225326592 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175626880641926, + "loss": 2.719, + "theoretical_loss": 3.580571788419575, + "tokens_seen": 1225392128 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175526579739218, + "loss": 2.5082, + "theoretical_loss": 3.5805542433259583, + "tokens_seen": 1225457664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 678803, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.4843032360076904, + "objective/train/theoretical_loss": 3.5805366994333125, + "objective/train/tokens_used": 1245983200, + "theoretical_loss": 3.5805366994333125, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031754262788365097, + "loss": 2.6552, + "theoretical_loss": 3.5805366994333125, + "tokens_seen": 1225523200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031753259779338015, + "loss": 2.7421, + "theoretical_loss": 3.5805191567414907, + "tokens_seen": 1225588736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031752256770310933, + "loss": 2.7794, + "theoretical_loss": 3.5805016152503466, + "tokens_seen": 1225654272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031751253761283857, + "loss": 2.7931, + "theoretical_loss": 3.580484074959734, + "tokens_seen": 1225719808 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003175025075225677, + "loss": 2.6884, + "theoretical_loss": 3.5804665358695074, + "tokens_seen": 1225785344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031749247743229693, + "loss": 2.5482, + "theoretical_loss": 3.5804489979795187, + "tokens_seen": 1225850880 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031748244734202606, + "loss": 2.7467, + "theoretical_loss": 3.580431461289623, + "tokens_seen": 1225916416 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174724172517553, + "loss": 2.7609, + "theoretical_loss": 3.5804139257996734, + "tokens_seen": 1225981952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174623871614845, + "loss": 2.5784, + "theoretical_loss": 3.580396391509524, + "tokens_seen": 1226047488 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031745235707121366, + "loss": 2.6765, + "theoretical_loss": 3.580378858419028, + "tokens_seen": 1226113024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031744232698094284, + "loss": 2.7612, + "theoretical_loss": 3.58036132652804, + "tokens_seen": 1226178560 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317432296890672, + "loss": 2.5738, + "theoretical_loss": 3.5803437958364137, + "tokens_seen": 1226244096 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003174222668004012, + "loss": 2.6951, + "theoretical_loss": 3.5803262663440023, + "tokens_seen": 1226309632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031741223671013044, + "loss": 2.6944, + "theoretical_loss": 3.58030873805066, + "tokens_seen": 1226375168 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031740220661985956, + "loss": 2.7802, + "theoretical_loss": 3.580291210956241, + "tokens_seen": 1226440704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173921765295888, + "loss": 2.5398, + "theoretical_loss": 3.5802736850605994, + "tokens_seen": 1226506240 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317382146439318, + "loss": 2.6482, + "theoretical_loss": 3.580256160363588, + "tokens_seen": 1226571776 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031737211634904716, + "loss": 2.7186, + "theoretical_loss": 3.5802386368650625, + "tokens_seen": 1226637312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031736208625877634, + "loss": 2.534, + "theoretical_loss": 3.5802211145648757, + "tokens_seen": 1226702848 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173520561685055, + "loss": 2.5662, + "theoretical_loss": 3.5802035934628815, + "tokens_seen": 1226768384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173420260782347, + "loss": 2.7673, + "theoretical_loss": 3.5801860735589344, + "tokens_seen": 1226833920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031733199598796394, + "loss": 2.5737, + "theoretical_loss": 3.5801685548528894, + "tokens_seen": 1226899456 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031732196589769307, + "loss": 2.7677, + "theoretical_loss": 3.5801510373445993, + "tokens_seen": 1226964992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003173119358074223, + "loss": 2.4772, + "theoretical_loss": 3.5801335210339182, + "tokens_seen": 1227030528 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031730190571715143, + "loss": 2.6551, + "theoretical_loss": 3.5801160059207007, + "tokens_seen": 1227096064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 679787, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.1799423694610596, + "objective/train/theoretical_loss": 3.580098492004802, + "objective/train/tokens_used": 1247621600, + "theoretical_loss": 3.580098492004802, + "tokens_seen": 1227161600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031729187562688066, + "loss": 2.7681, + "theoretical_loss": 3.580098492004802, + "tokens_seen": 1227161600 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031728184553660984, + "loss": 2.6133, + "theoretical_loss": 3.580080979286074, + "tokens_seen": 1227227136 + }, + { + "epoch": 0.37, + "learning_rate": 0.000317271815446339, + "loss": 2.7949, + "theoretical_loss": 3.580063467764373, + "tokens_seen": 1227292672 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172617853560682, + "loss": 2.4662, + "theoretical_loss": 3.5800459574395527, + "tokens_seen": 1227358208 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172517552657974, + "loss": 2.7973, + "theoretical_loss": 3.5800284483114666, + "tokens_seen": 1227423744 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031724172517552657, + "loss": 2.4858, + "theoretical_loss": 3.58001094037997, + "tokens_seen": 1227489280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003172316950852558, + "loss": 2.7632, + "theoretical_loss": 3.5799934336449173, + "tokens_seen": 1227554816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031722166499498493, + "loss": 2.4781, + "theoretical_loss": 3.579975928106162, + "tokens_seen": 1227620352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031721163490471417, + "loss": 2.825, + "theoretical_loss": 3.579958423763559, + "tokens_seen": 1227685888 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031720160481444335, + "loss": 2.662, + "theoretical_loss": 3.579940920616963, + "tokens_seen": 1227751424 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031719157472417253, + "loss": 2.6949, + "theoretical_loss": 3.579923418666228, + "tokens_seen": 1227816960 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171815446339017, + "loss": 2.7449, + "theoretical_loss": 3.5799059179112076, + "tokens_seen": 1227882496 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171715145436309, + "loss": 2.7963, + "theoretical_loss": 3.5798884183517585, + "tokens_seen": 1227948032 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031716148445336007, + "loss": 2.7011, + "theoretical_loss": 3.5798709199877337, + "tokens_seen": 1228013568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171514543630893, + "loss": 2.8564, + "theoretical_loss": 3.5798534228189878, + "tokens_seen": 1228079104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031714142427281843, + "loss": 2.6509, + "theoretical_loss": 3.5798359268453757, + "tokens_seen": 1228144640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031713139418254767, + "loss": 2.5021, + "theoretical_loss": 3.5798184320667525, + "tokens_seen": 1228210176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171213640922768, + "loss": 2.6051, + "theoretical_loss": 3.579800938482972, + "tokens_seen": 1228275712 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031711133400200603, + "loss": 2.8244, + "theoretical_loss": 3.579783446093889, + "tokens_seen": 1228341248 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003171013039117352, + "loss": 2.8286, + "theoretical_loss": 3.579765954899358, + "tokens_seen": 1228406784 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170912738214644, + "loss": 2.8978, + "theoretical_loss": 3.579748464899234, + "tokens_seen": 1228472320 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170812437311936, + "loss": 2.6003, + "theoretical_loss": 3.579730976093372, + "tokens_seen": 1228537856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170712136409228, + "loss": 2.6917, + "theoretical_loss": 3.5797134884816266, + "tokens_seen": 1228603392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031706118355065194, + "loss": 2.6362, + "theoretical_loss": 3.5796960020638524, + "tokens_seen": 1228668928 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170511534603812, + "loss": 2.604, + "theoretical_loss": 3.5796785168399037, + "tokens_seen": 1228734464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 680312, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.374969482421875, + "objective/train/theoretical_loss": 3.5796610328096365, + "objective/train/tokens_used": 1249260000, + "theoretical_loss": 3.5796610328096365, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170411233701103, + "loss": 2.6887, + "theoretical_loss": 3.5796610328096365, + "tokens_seen": 1228800000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031703109327983954, + "loss": 2.7248, + "theoretical_loss": 3.5796435499729045, + "tokens_seen": 1228865536 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170210631895687, + "loss": 2.8177, + "theoretical_loss": 3.5796260683295635, + "tokens_seen": 1228931072 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170110330992979, + "loss": 2.6824, + "theoretical_loss": 3.579608587879468, + "tokens_seen": 1228996608 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003170010030090271, + "loss": 2.9693, + "theoretical_loss": 3.5795911086224725, + "tokens_seen": 1229062144 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031699097291875626, + "loss": 2.7433, + "theoretical_loss": 3.579573630558433, + "tokens_seen": 1229127680 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031698094282848544, + "loss": 2.5788, + "theoretical_loss": 3.5795561536872036, + "tokens_seen": 1229193216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169709127382147, + "loss": 2.7061, + "theoretical_loss": 3.57953867800864, + "tokens_seen": 1229258752 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169608826479438, + "loss": 2.6546, + "theoretical_loss": 3.5795212035225963, + "tokens_seen": 1229324288 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031695085255767304, + "loss": 2.3916, + "theoretical_loss": 3.579503730228928, + "tokens_seen": 1229389824 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031694082246740217, + "loss": 2.5509, + "theoretical_loss": 3.57948625812749, + "tokens_seen": 1229455360 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169307923771314, + "loss": 2.6121, + "theoretical_loss": 3.5794687872181385, + "tokens_seen": 1229520896 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003169207622868606, + "loss": 2.3906, + "theoretical_loss": 3.5794513175007276, + "tokens_seen": 1229586432 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031691073219658976, + "loss": 2.6725, + "theoretical_loss": 3.5794338489751123, + "tokens_seen": 1229651968 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031690070210631894, + "loss": 2.7202, + "theoretical_loss": 3.579416381641148, + "tokens_seen": 1229717504 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168906720160482, + "loss": 2.8365, + "theoretical_loss": 3.5793989154986905, + "tokens_seen": 1229783040 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168806419257773, + "loss": 2.6397, + "theoretical_loss": 3.5793814505475945, + "tokens_seen": 1229848576 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031687061183550654, + "loss": 2.7826, + "theoretical_loss": 3.5793639867877154, + "tokens_seen": 1229914112 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031686058174523567, + "loss": 2.472, + "theoretical_loss": 3.5793465242189075, + "tokens_seen": 1229979648 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168505516549649, + "loss": 2.6565, + "theoretical_loss": 3.579329062841028, + "tokens_seen": 1230045184 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003168405215646941, + "loss": 2.6165, + "theoretical_loss": 3.579311602653931, + "tokens_seen": 1230110720 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031683049147442327, + "loss": 2.807, + "theoretical_loss": 3.5792941436574726, + "tokens_seen": 1230176256 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031682046138415245, + "loss": 2.6677, + "theoretical_loss": 3.579276685851507, + "tokens_seen": 1230241792 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031681043129388163, + "loss": 2.5583, + "theoretical_loss": 3.5792592292358902, + "tokens_seen": 1230307328 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031680040120361086, + "loss": 2.8452, + "theoretical_loss": 3.5792417738104785, + "tokens_seen": 1230372864 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 681644, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.6463749408721924, + "objective/train/theoretical_loss": 3.579224319575126, + "objective/train/tokens_used": 1250898400, + "theoretical_loss": 3.579224319575126, + "tokens_seen": 1230438400 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031679037111334004, + "loss": 2.5301, + "theoretical_loss": 3.579224319575126, + "tokens_seen": 1230438400 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167803410230692, + "loss": 2.6175, + "theoretical_loss": 3.579206866529689, + "tokens_seen": 1230503936 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167703109327984, + "loss": 2.7769, + "theoretical_loss": 3.5791894146740226, + "tokens_seen": 1230569472 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003167602808425276, + "loss": 2.7163, + "theoretical_loss": 3.579171964007983, + "tokens_seen": 1230635008 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031675025075225677, + "loss": 2.658, + "theoretical_loss": 3.579154514531425, + "tokens_seen": 1230700544 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316740220661986, + "loss": 2.7854, + "theoretical_loss": 3.579137066244204, + "tokens_seen": 1230766080 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031673019057171513, + "loss": 2.747, + "theoretical_loss": 3.5791196191461774, + "tokens_seen": 1230831616 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031672016048144437, + "loss": 2.5688, + "theoretical_loss": 3.5791021732371986, + "tokens_seen": 1230897152 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031671013039117355, + "loss": 2.8399, + "theoretical_loss": 3.5790847285171243, + "tokens_seen": 1230962688 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031670010030090273, + "loss": 2.5067, + "theoretical_loss": 3.5790672849858103, + "tokens_seen": 1231028224 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166900702106319, + "loss": 2.7153, + "theoretical_loss": 3.579049842643112, + "tokens_seen": 1231093760 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166800401203611, + "loss": 2.6044, + "theoretical_loss": 3.5790324014888855, + "tokens_seen": 1231159296 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031667001003009027, + "loss": 2.719, + "theoretical_loss": 3.5790149615229856, + "tokens_seen": 1231224832 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166599799398195, + "loss": 2.5149, + "theoretical_loss": 3.5789975227452695, + "tokens_seen": 1231290368 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031664994984954863, + "loss": 2.7169, + "theoretical_loss": 3.5789800851555924, + "tokens_seen": 1231355904 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031663991975927787, + "loss": 2.51, + "theoretical_loss": 3.5789626487538095, + "tokens_seen": 1231421440 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316629889669007, + "loss": 2.6168, + "theoretical_loss": 3.5789452135397775, + "tokens_seen": 1231486976 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031661985957873623, + "loss": 2.601, + "theoretical_loss": 3.578927779513352, + "tokens_seen": 1231552512 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003166098294884654, + "loss": 2.7909, + "theoretical_loss": 3.578910346674389, + "tokens_seen": 1231618048 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165997993981946, + "loss": 2.6013, + "theoretical_loss": 3.578892915022744, + "tokens_seen": 1231683584 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165897693079238, + "loss": 2.7524, + "theoretical_loss": 3.5788754845582735, + "tokens_seen": 1231749120 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316579739217653, + "loss": 2.8369, + "theoretical_loss": 3.5788580552808336, + "tokens_seen": 1231814656 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031656970912738214, + "loss": 2.6386, + "theoretical_loss": 3.5788406271902797, + "tokens_seen": 1231880192 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165596790371114, + "loss": 2.6326, + "theoretical_loss": 3.578823200286468, + "tokens_seen": 1231945728 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165496489468405, + "loss": 2.5907, + "theoretical_loss": 3.5788057745692554, + "tokens_seen": 1232011264 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 682283, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9270589351654053, + "objective/train/theoretical_loss": 3.578788350038497, + "objective/train/tokens_used": 1252536800, + "theoretical_loss": 3.578788350038497, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031653961885656974, + "loss": 2.7132, + "theoretical_loss": 3.578788350038497, + "tokens_seen": 1232076800 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165295887662989, + "loss": 2.8002, + "theoretical_loss": 3.578770926694049, + "tokens_seen": 1232142336 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165195586760281, + "loss": 2.4558, + "theoretical_loss": 3.5787535045357686, + "tokens_seen": 1232207872 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003165095285857573, + "loss": 2.5559, + "theoretical_loss": 3.57873608356351, + "tokens_seen": 1232273408 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031649949849548646, + "loss": 2.4907, + "theoretical_loss": 3.5787186637771313, + "tokens_seen": 1232338944 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031648946840521564, + "loss": 2.6496, + "theoretical_loss": 3.5787012451764877, + "tokens_seen": 1232404480 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164794383149449, + "loss": 2.7241, + "theoretical_loss": 3.578683827761436, + "tokens_seen": 1232470016 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316469408224674, + "loss": 2.7307, + "theoretical_loss": 3.578666411531832, + "tokens_seen": 1232535552 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031645937813440324, + "loss": 2.5383, + "theoretical_loss": 3.578648996487532, + "tokens_seen": 1232601088 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031644934804413237, + "loss": 2.729, + "theoretical_loss": 3.5786315826283923, + "tokens_seen": 1232666624 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164393179538616, + "loss": 2.7778, + "theoretical_loss": 3.57861416995427, + "tokens_seen": 1232732160 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003164292878635908, + "loss": 2.5696, + "theoretical_loss": 3.5785967584650207, + "tokens_seen": 1232797696 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031641925777331996, + "loss": 2.4963, + "theoretical_loss": 3.578579348160501, + "tokens_seen": 1232863232 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031640922768304914, + "loss": 2.4366, + "theoretical_loss": 3.5785619390405667, + "tokens_seen": 1232928768 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163991975927784, + "loss": 2.8987, + "theoretical_loss": 3.5785445311050754, + "tokens_seen": 1232994304 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163891675025075, + "loss": 2.3145, + "theoretical_loss": 3.578527124353883, + "tokens_seen": 1233059840 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031637913741223674, + "loss": 2.6965, + "theoretical_loss": 3.5785097187868455, + "tokens_seen": 1233125376 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031636910732196587, + "loss": 2.6582, + "theoretical_loss": 3.57849231440382, + "tokens_seen": 1233190912 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163590772316951, + "loss": 2.5731, + "theoretical_loss": 3.5784749112046637, + "tokens_seen": 1233256448 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003163490471414243, + "loss": 2.5886, + "theoretical_loss": 3.578457509189232, + "tokens_seen": 1233321984 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031633901705115347, + "loss": 2.6165, + "theoretical_loss": 3.578440108357382, + "tokens_seen": 1233387520 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031632898696088265, + "loss": 2.663, + "theoretical_loss": 3.57842270870897, + "tokens_seen": 1233453056 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031631895687061183, + "loss": 2.7879, + "theoretical_loss": 3.578405310243853, + "tokens_seen": 1233518592 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316308926780341, + "loss": 2.4878, + "theoretical_loss": 3.578387912961887, + "tokens_seen": 1233584128 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031629889669007024, + "loss": 2.8083, + "theoretical_loss": 3.57837051686293, + "tokens_seen": 1233649664 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 683327, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.9415130615234375, + "objective/train/theoretical_loss": 3.5783531219468374, + "objective/train/tokens_used": 1254175200, + "theoretical_loss": 3.5783531219468374, + "tokens_seen": 1233715200 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031628886659979937, + "loss": 2.7898, + "theoretical_loss": 3.5783531219468374, + "tokens_seen": 1233715200 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162788365095286, + "loss": 2.555, + "theoretical_loss": 3.5783357282134665, + "tokens_seen": 1233780736 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031626880641925773, + "loss": 2.7074, + "theoretical_loss": 3.578318335662675, + "tokens_seen": 1233846272 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031625877632898697, + "loss": 2.7915, + "theoretical_loss": 3.5783009442943174, + "tokens_seen": 1233911808 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031624874623871615, + "loss": 2.6769, + "theoretical_loss": 3.5782835541082525, + "tokens_seen": 1233977344 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031623871614844533, + "loss": 2.595, + "theoretical_loss": 3.5782661651043366, + "tokens_seen": 1234042880 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162286860581745, + "loss": 2.6416, + "theoretical_loss": 3.5782487772824263, + "tokens_seen": 1234108416 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031621865596790375, + "loss": 2.8331, + "theoretical_loss": 3.5782313906423786, + "tokens_seen": 1234173952 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003162086258776329, + "loss": 2.4344, + "theoretical_loss": 3.5782140051840505, + "tokens_seen": 1234239488 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161985957873621, + "loss": 2.4601, + "theoretical_loss": 3.578196620907299, + "tokens_seen": 1234305024 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031618856569709124, + "loss": 2.7753, + "theoretical_loss": 3.578179237811981, + "tokens_seen": 1234370560 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031617853560682047, + "loss": 2.722, + "theoretical_loss": 3.578161855897953, + "tokens_seen": 1234436096 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031616850551654965, + "loss": 2.5857, + "theoretical_loss": 3.5781444751650735, + "tokens_seen": 1234501632 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031615847542627883, + "loss": 2.5077, + "theoretical_loss": 3.5781270956131976, + "tokens_seen": 1234567168 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316148445336008, + "loss": 2.5656, + "theoretical_loss": 3.5781097172421834, + "tokens_seen": 1234632704 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161384152457372, + "loss": 2.5073, + "theoretical_loss": 3.5780923400518883, + "tokens_seen": 1234698240 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161283851554664, + "loss": 2.6107, + "theoretical_loss": 3.5780749640421687, + "tokens_seen": 1234763776 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003161183550651956, + "loss": 2.5824, + "theoretical_loss": 3.5780575892128823, + "tokens_seen": 1234829312 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031610832497492474, + "loss": 2.631, + "theoretical_loss": 3.5780402155638855, + "tokens_seen": 1234894848 + }, + { + "epoch": 0.37, + "learning_rate": 0.000316098294884654, + "loss": 2.4193, + "theoretical_loss": 3.5780228430950363, + "tokens_seen": 1234960384 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160882647943831, + "loss": 2.7855, + "theoretical_loss": 3.5780054718061916, + "tokens_seen": 1235025920 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031607823470411234, + "loss": 2.6025, + "theoretical_loss": 3.5779881016972093, + "tokens_seen": 1235091456 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160682046138415, + "loss": 2.8302, + "theoretical_loss": 3.577970732767945, + "tokens_seen": 1235156992 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160581745235707, + "loss": 2.5224, + "theoretical_loss": 3.5779533650182573, + "tokens_seen": 1235222528 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031604814443329994, + "loss": 2.7319, + "theoretical_loss": 3.5779359984480035, + "tokens_seen": 1235288064 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 684175, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 3.004976272583008, + "objective/train/theoretical_loss": 3.5779186330570405, + "objective/train/tokens_used": 1255813600, + "theoretical_loss": 3.5779186330570405, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160381143430291, + "loss": 2.6376, + "theoretical_loss": 3.5779186330570405, + "tokens_seen": 1235353600 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160280842527583, + "loss": 2.6485, + "theoretical_loss": 3.577901268845226, + "tokens_seen": 1235419136 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003160180541624875, + "loss": 2.6592, + "theoretical_loss": 3.5778839058124166, + "tokens_seen": 1235484672 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031600802407221666, + "loss": 2.5898, + "theoretical_loss": 3.577866543958471, + "tokens_seen": 1235550208 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031599799398194584, + "loss": 2.7968, + "theoretical_loss": 3.577849183283245, + "tokens_seen": 1235615744 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159879638916751, + "loss": 2.7619, + "theoretical_loss": 3.577831823786598, + "tokens_seen": 1235681280 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159779338014042, + "loss": 2.6657, + "theoretical_loss": 3.577814465468386, + "tokens_seen": 1235746816 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031596790371113344, + "loss": 2.5929, + "theoretical_loss": 3.5777971083284674, + "tokens_seen": 1235812352 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031595787362086257, + "loss": 2.6074, + "theoretical_loss": 3.577779752366699, + "tokens_seen": 1235877888 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159478435305918, + "loss": 2.5263, + "theoretical_loss": 3.577762397582939, + "tokens_seen": 1235943424 + }, + { + "epoch": 0.37, + "learning_rate": 0.000315937813440321, + "loss": 2.5976, + "theoretical_loss": 3.5777450439770444, + "tokens_seen": 1236008960 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031592778335005016, + "loss": 2.7342, + "theoretical_loss": 3.5777276915488736, + "tokens_seen": 1236074496 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031591775325977934, + "loss": 2.6336, + "theoretical_loss": 3.5777103402982835, + "tokens_seen": 1236140032 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003159077231695086, + "loss": 2.8675, + "theoretical_loss": 3.577692990225132, + "tokens_seen": 1236205568 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158976930792377, + "loss": 2.791, + "theoretical_loss": 3.577675641329277, + "tokens_seen": 1236271104 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031588766298896694, + "loss": 2.7231, + "theoretical_loss": 3.5776582936105754, + "tokens_seen": 1236336640 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031587763289869607, + "loss": 2.575, + "theoretical_loss": 3.577640947068886, + "tokens_seen": 1236402176 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158676028084253, + "loss": 2.8744, + "theoretical_loss": 3.577623601704066, + "tokens_seen": 1236467712 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158575727181545, + "loss": 2.7638, + "theoretical_loss": 3.5776062575159733, + "tokens_seen": 1236533248 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031584754262788367, + "loss": 2.7146, + "theoretical_loss": 3.577588914504466, + "tokens_seen": 1236598784 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031583751253761285, + "loss": 2.8056, + "theoretical_loss": 3.5775715726694015, + "tokens_seen": 1236664320 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031582748244734203, + "loss": 2.5687, + "theoretical_loss": 3.5775542320106375, + "tokens_seen": 1236729856 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003158174523570712, + "loss": 2.8633, + "theoretical_loss": 3.577536892528032, + "tokens_seen": 1236795392 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031580742226680044, + "loss": 2.4725, + "theoretical_loss": 3.577519554221444, + "tokens_seen": 1236860928 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031579739217652957, + "loss": 2.7498, + "theoretical_loss": 3.5775022170907302, + "tokens_seen": 1236926464 + }, + { + "epoch": 0.37, + "objective/train/docs_used": 684175, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.575035572052002, + "objective/train/theoretical_loss": 3.5774848811357485, + "objective/train/tokens_used": 1257452000, + "theoretical_loss": 3.5774848811357485, + "tokens_seen": 1236992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157873620862588, + "loss": 2.7459, + "theoretical_loss": 3.5774848811357485, + "tokens_seen": 1236992000 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031577733199598793, + "loss": 2.5454, + "theoretical_loss": 3.5774675463563574, + "tokens_seen": 1237057536 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031576730190571717, + "loss": 2.6331, + "theoretical_loss": 3.5774502127524146, + "tokens_seen": 1237123072 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031575727181544635, + "loss": 2.7631, + "theoretical_loss": 3.5774328803237787, + "tokens_seen": 1237188608 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031574724172517553, + "loss": 2.841, + "theoretical_loss": 3.577415549070307, + "tokens_seen": 1237254144 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157372116349047, + "loss": 2.6371, + "theoretical_loss": 3.577398218991858, + "tokens_seen": 1237319680 + }, + { + "epoch": 0.37, + "learning_rate": 0.00031572718154463395, + "loss": 2.8772, + "theoretical_loss": 3.57738089008829, + "tokens_seen": 1237385216 + }, + { + "epoch": 0.37, + "learning_rate": 0.0003157171514543631, + "loss": 2.8408, + "theoretical_loss": 3.577363562359461, + "tokens_seen": 1237450752 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003157071213640923, + "loss": 2.6924, + "theoretical_loss": 3.5773462358052286, + "tokens_seen": 1237516288 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031569709127382144, + "loss": 2.6864, + "theoretical_loss": 3.5773289104254515, + "tokens_seen": 1237581824 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031568706118355067, + "loss": 2.7785, + "theoretical_loss": 3.577311586219988, + "tokens_seen": 1237647360 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031567703109327985, + "loss": 2.5432, + "theoretical_loss": 3.577294263188696, + "tokens_seen": 1237712896 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031566700100300903, + "loss": 2.5293, + "theoretical_loss": 3.577276941331434, + "tokens_seen": 1237778432 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156569709127382, + "loss": 2.6125, + "theoretical_loss": 3.57725962064806, + "tokens_seen": 1237843968 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156469408224674, + "loss": 2.5755, + "theoretical_loss": 3.5772423011384324, + "tokens_seen": 1237909504 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156369107321966, + "loss": 2.6375, + "theoretical_loss": 3.5772249828024103, + "tokens_seen": 1237975040 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156268806419258, + "loss": 2.4348, + "theoretical_loss": 3.577207665639851, + "tokens_seen": 1238040576 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031561685055165494, + "loss": 2.7653, + "theoretical_loss": 3.5771903496506128, + "tokens_seen": 1238106112 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003156068204613842, + "loss": 2.8229, + "theoretical_loss": 3.577173034834555, + "tokens_seen": 1238171648 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155967903711133, + "loss": 2.6971, + "theoretical_loss": 3.5771557211915352, + "tokens_seen": 1238237184 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031558676028084254, + "loss": 2.7564, + "theoretical_loss": 3.5771384087214124, + "tokens_seen": 1238302720 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155767301905717, + "loss": 2.4933, + "theoretical_loss": 3.577121097424045, + "tokens_seen": 1238368256 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155667001003009, + "loss": 2.5807, + "theoretical_loss": 3.5771037872992917, + "tokens_seen": 1238433792 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155566700100301, + "loss": 2.5816, + "theoretical_loss": 3.57708647834701, + "tokens_seen": 1238499328 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155466399197593, + "loss": 2.9417, + "theoretical_loss": 3.5770691705670594, + "tokens_seen": 1238564864 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 685182, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.7375147342681885, + "objective/train/theoretical_loss": 3.5770518639592983, + "objective/train/tokens_used": 1259090400, + "theoretical_loss": 3.5770518639592983, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031553660982948844, + "loss": 2.6052, + "theoretical_loss": 3.5770518639592983, + "tokens_seen": 1238630400 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155265797392177, + "loss": 2.7781, + "theoretical_loss": 3.577034558523585, + "tokens_seen": 1238695936 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003155165496489468, + "loss": 2.8369, + "theoretical_loss": 3.577017254259779, + "tokens_seen": 1238761472 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031550651955867604, + "loss": 2.7518, + "theoretical_loss": 3.5769999511677377, + "tokens_seen": 1238827008 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154964894684052, + "loss": 2.5688, + "theoretical_loss": 3.576982649247321, + "tokens_seen": 1238892544 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154864593781344, + "loss": 2.7044, + "theoretical_loss": 3.576965348498386, + "tokens_seen": 1238958080 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154764292878636, + "loss": 2.9206, + "theoretical_loss": 3.5769480489207934, + "tokens_seen": 1239023616 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031546639919759277, + "loss": 2.8187, + "theoretical_loss": 3.5769307505144003, + "tokens_seen": 1239089152 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031545636910732195, + "loss": 2.5674, + "theoretical_loss": 3.576913453279066, + "tokens_seen": 1239154688 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154463390170512, + "loss": 2.7482, + "theoretical_loss": 3.57689615721465, + "tokens_seen": 1239220224 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154363089267803, + "loss": 2.8608, + "theoretical_loss": 3.57687886232101, + "tokens_seen": 1239285760 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031542627883650954, + "loss": 2.6337, + "theoretical_loss": 3.5768615685980056, + "tokens_seen": 1239351296 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154162487462387, + "loss": 2.7525, + "theoretical_loss": 3.5768442760454953, + "tokens_seen": 1239416832 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003154062186559679, + "loss": 2.6064, + "theoretical_loss": 3.5768269846633385, + "tokens_seen": 1239482368 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153961885656971, + "loss": 2.6995, + "theoretical_loss": 3.576809694451393, + "tokens_seen": 1239547904 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031538615847542627, + "loss": 2.5633, + "theoretical_loss": 3.5767924054095195, + "tokens_seen": 1239613440 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031537612838515545, + "loss": 2.7152, + "theoretical_loss": 3.5767751175375753, + "tokens_seen": 1239678976 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153660982948847, + "loss": 2.8287, + "theoretical_loss": 3.5767578308354198, + "tokens_seen": 1239744512 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153560682046138, + "loss": 2.8507, + "theoretical_loss": 3.5767405453029126, + "tokens_seen": 1239810048 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031534603811434305, + "loss": 2.799, + "theoretical_loss": 3.5767232609399127, + "tokens_seen": 1239875584 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153360080240722, + "loss": 2.725, + "theoretical_loss": 3.576705977746278, + "tokens_seen": 1239941120 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153259779338014, + "loss": 2.6619, + "theoretical_loss": 3.5766886957218693, + "tokens_seen": 1240006656 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003153159478435306, + "loss": 2.8006, + "theoretical_loss": 3.576671414866544, + "tokens_seen": 1240072192 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031530591775325977, + "loss": 2.6383, + "theoretical_loss": 3.5766541351801626, + "tokens_seen": 1240137728 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315295887662989, + "loss": 2.5801, + "theoretical_loss": 3.576636856662584, + "tokens_seen": 1240203264 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 685844, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.0006914138793945, + "objective/train/theoretical_loss": 3.5766195793136664, + "objective/train/tokens_used": 1260728800, + "theoretical_loss": 3.5766195793136664, + "tokens_seen": 1240268800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031528585757271813, + "loss": 2.4128, + "theoretical_loss": 3.5766195793136664, + "tokens_seen": 1240268800 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031527582748244737, + "loss": 2.6779, + "theoretical_loss": 3.5766023031332703, + "tokens_seen": 1240334336 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031526579739217655, + "loss": 2.8779, + "theoretical_loss": 3.5765850281212543, + "tokens_seen": 1240399872 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031525576730190573, + "loss": 2.5602, + "theoretical_loss": 3.5765677542774776, + "tokens_seen": 1240465408 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152457372116349, + "loss": 2.7543, + "theoretical_loss": 3.5765504816018, + "tokens_seen": 1240530944 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031523570712136415, + "loss": 2.7578, + "theoretical_loss": 3.57653321009408, + "tokens_seen": 1240596480 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152256770310933, + "loss": 2.6434, + "theoretical_loss": 3.5765159397541773, + "tokens_seen": 1240662016 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003152156469408225, + "loss": 2.6226, + "theoretical_loss": 3.5764986705819517, + "tokens_seen": 1240727552 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031520561685055164, + "loss": 2.9745, + "theoretical_loss": 3.5764814025772615, + "tokens_seen": 1240793088 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031519558676028087, + "loss": 2.8188, + "theoretical_loss": 3.5764641357399674, + "tokens_seen": 1240858624 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031518555667001005, + "loss": 2.7181, + "theoretical_loss": 3.576446870069928, + "tokens_seen": 1240924160 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031517552657973924, + "loss": 2.8341, + "theoretical_loss": 3.576429605567003, + "tokens_seen": 1240989696 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151654964894684, + "loss": 2.7724, + "theoretical_loss": 3.576412342231052, + "tokens_seen": 1241055232 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151554663991976, + "loss": 2.6043, + "theoretical_loss": 3.576395080061934, + "tokens_seen": 1241120768 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151454363089268, + "loss": 2.6358, + "theoretical_loss": 3.576377819059509, + "tokens_seen": 1241186304 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315135406218656, + "loss": 2.8804, + "theoretical_loss": 3.5763605592236365, + "tokens_seen": 1241251840 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031512537612838514, + "loss": 2.7021, + "theoretical_loss": 3.5763433005541767, + "tokens_seen": 1241317376 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151153460381144, + "loss": 2.3519, + "theoretical_loss": 3.5763260430509876, + "tokens_seen": 1241382912 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003151053159478435, + "loss": 2.8287, + "theoretical_loss": 3.57630878671393, + "tokens_seen": 1241448448 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031509528585757274, + "loss": 2.3984, + "theoretical_loss": 3.5762915315428634, + "tokens_seen": 1241513984 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150852557673019, + "loss": 2.6735, + "theoretical_loss": 3.576274277537647, + "tokens_seen": 1241579520 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150752256770311, + "loss": 2.7345, + "theoretical_loss": 3.5762570246981413, + "tokens_seen": 1241645056 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150651955867603, + "loss": 2.6689, + "theoretical_loss": 3.5762397730242053, + "tokens_seen": 1241710592 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150551654964895, + "loss": 2.8214, + "theoretical_loss": 3.5762225225156987, + "tokens_seen": 1241776128 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031504513540621864, + "loss": 2.8196, + "theoretical_loss": 3.576205273172482, + "tokens_seen": 1241841664 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 687001, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8483855724334717, + "objective/train/theoretical_loss": 3.5761880249944147, + "objective/train/tokens_used": 1262367200, + "theoretical_loss": 3.5761880249944147, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150351053159479, + "loss": 2.6797, + "theoretical_loss": 3.5761880249944147, + "tokens_seen": 1241907200 + }, + { + "epoch": 0.38, + "learning_rate": 0.000315025075225677, + "loss": 2.8243, + "theoretical_loss": 3.576170777981356, + "tokens_seen": 1241972736 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031501504513540624, + "loss": 2.812, + "theoretical_loss": 3.5761535321331666, + "tokens_seen": 1242038272 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003150050150451354, + "loss": 2.6161, + "theoretical_loss": 3.5761362874497062, + "tokens_seen": 1242103808 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149949849548646, + "loss": 2.5499, + "theoretical_loss": 3.5761190439308344, + "tokens_seen": 1242169344 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149849548645938, + "loss": 2.7484, + "theoretical_loss": 3.576101801576411, + "tokens_seen": 1242234880 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031497492477432297, + "loss": 2.7865, + "theoretical_loss": 3.5760845603862963, + "tokens_seen": 1242300416 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031496489468405215, + "loss": 2.7135, + "theoretical_loss": 3.57606732036035, + "tokens_seen": 1242365952 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149548645937814, + "loss": 2.8205, + "theoretical_loss": 3.576050081498432, + "tokens_seen": 1242431488 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149448345035105, + "loss": 2.6539, + "theoretical_loss": 3.5760328438004025, + "tokens_seen": 1242497024 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031493480441323974, + "loss": 2.5831, + "theoretical_loss": 3.5760156072661218, + "tokens_seen": 1242562560 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149247743229689, + "loss": 2.7412, + "theoretical_loss": 3.57599837189545, + "tokens_seen": 1242628096 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149147442326981, + "loss": 2.7729, + "theoretical_loss": 3.575981137688246, + "tokens_seen": 1242693632 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003149047141424273, + "loss": 2.5824, + "theoretical_loss": 3.5759639046443716, + "tokens_seen": 1242759168 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031489468405215647, + "loss": 2.6411, + "theoretical_loss": 3.575946672763686, + "tokens_seen": 1242824704 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031488465396188565, + "loss": 2.7375, + "theoretical_loss": 3.575929442046049, + "tokens_seen": 1242890240 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148746238716149, + "loss": 2.7509, + "theoretical_loss": 3.575912212491322, + "tokens_seen": 1242955776 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314864593781344, + "loss": 2.7655, + "theoretical_loss": 3.575894984099364, + "tokens_seen": 1243021312 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031485456369107325, + "loss": 2.8217, + "theoretical_loss": 3.575877756870036, + "tokens_seen": 1243086848 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148445336008024, + "loss": 2.6609, + "theoretical_loss": 3.575860530803198, + "tokens_seen": 1243152384 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148345035105316, + "loss": 2.8145, + "theoretical_loss": 3.57584330589871, + "tokens_seen": 1243217920 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003148244734202608, + "loss": 2.6594, + "theoretical_loss": 3.575826082156432, + "tokens_seen": 1243283456 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031481444332998997, + "loss": 2.5629, + "theoretical_loss": 3.5758088595762256, + "tokens_seen": 1243348992 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031480441323971915, + "loss": 2.5686, + "theoretical_loss": 3.57579163815795, + "tokens_seen": 1243414528 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031479438314944833, + "loss": 2.7092, + "theoretical_loss": 3.5757744179014668, + "tokens_seen": 1243480064 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 688085, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.8447015285491943, + "objective/train/theoretical_loss": 3.575757198806635, + "objective/train/tokens_used": 1264005600, + "theoretical_loss": 3.575757198806635, + "tokens_seen": 1243545600 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147843530591775, + "loss": 2.9024, + "theoretical_loss": 3.575757198806635, + "tokens_seen": 1243545600 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031477432296890675, + "loss": 2.7652, + "theoretical_loss": 3.5757399808733155, + "tokens_seen": 1243611136 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147642928786359, + "loss": 2.8517, + "theoretical_loss": 3.575722764101369, + "tokens_seen": 1243676672 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147542627883651, + "loss": 2.7982, + "theoretical_loss": 3.575705548490656, + "tokens_seen": 1243742208 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147442326980943, + "loss": 2.4399, + "theoretical_loss": 3.5756883340410366, + "tokens_seen": 1243807744 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003147342026078235, + "loss": 2.4896, + "theoretical_loss": 3.5756711207523715, + "tokens_seen": 1243873280 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031472417251755266, + "loss": 2.6552, + "theoretical_loss": 3.5756539086245214, + "tokens_seen": 1243938816 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031471414242728184, + "loss": 2.649, + "theoretical_loss": 3.575636697657347, + "tokens_seen": 1244004352 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314704112337011, + "loss": 2.7111, + "theoretical_loss": 3.5756194878507084, + "tokens_seen": 1244069888 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031469408224674025, + "loss": 2.6986, + "theoretical_loss": 3.5756022792044666, + "tokens_seen": 1244135424 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146840521564694, + "loss": 2.7551, + "theoretical_loss": 3.5755850717184816, + "tokens_seen": 1244200960 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146740220661986, + "loss": 2.7036, + "theoretical_loss": 3.575567865392615, + "tokens_seen": 1244266496 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031466399197592774, + "loss": 2.7044, + "theoretical_loss": 3.5755506602267273, + "tokens_seen": 1244332032 + }, + { + "epoch": 0.38, + "learning_rate": 0.000314653961885657, + "loss": 2.715, + "theoretical_loss": 3.575533456220678, + "tokens_seen": 1244397568 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031464393179538616, + "loss": 2.6174, + "theoretical_loss": 3.57551625337433, + "tokens_seen": 1244463104 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031463390170511534, + "loss": 2.6853, + "theoretical_loss": 3.575499051687542, + "tokens_seen": 1244528640 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146238716148445, + "loss": 2.8587, + "theoretical_loss": 3.575481851160176, + "tokens_seen": 1244594176 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146138415245737, + "loss": 2.8025, + "theoretical_loss": 3.5754646517920925, + "tokens_seen": 1244659712 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003146038114343029, + "loss": 2.8209, + "theoretical_loss": 3.575447453583152, + "tokens_seen": 1244725248 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145937813440321, + "loss": 2.5335, + "theoretical_loss": 3.575430256533216, + "tokens_seen": 1244790784 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031458375125376125, + "loss": 2.5279, + "theoretical_loss": 3.575413060642145, + "tokens_seen": 1244856320 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145737211634905, + "loss": 2.6258, + "theoretical_loss": 3.5753958659097993, + "tokens_seen": 1244921856 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145636910732197, + "loss": 2.6978, + "theoretical_loss": 3.5753786723360412, + "tokens_seen": 1244987392 + }, + { + "epoch": 0.38, + "learning_rate": 0.00031455366098294884, + "loss": 2.6572, + "theoretical_loss": 3.5753614799207307, + "tokens_seen": 1245052928 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145436308926781, + "loss": 2.9123, + "theoretical_loss": 3.5753442886637288, + "tokens_seen": 1245118464 + }, + { + "epoch": 0.38, + "objective/train/docs_used": 688769, + "objective/train/instantaneous_batch_size": 16, + "objective/train/instantaneous_microbatch_size": 16384, + "objective/train/original_loss": 2.5056591033935547, + "objective/train/theoretical_loss": 3.5753270985648973, + "objective/train/tokens_used": 1265644000, + "theoretical_loss": 3.5753270985648973, + "tokens_seen": 1245184000 + }, + { + "epoch": 0.38, + "learning_rate": 0.0003145336008024072, + "loss": 2.555, + "theoretical_loss": 3.5753270985648973, + "tokens_seen": 1245184000 + } + ], + "max_steps": 50354, + "num_train_epochs": 9223372036854775807, + "total_flos": 6.35462221824e+17, + "trial_name": null, + "trial_params": null +}